Merge pull request #1 from dualfroz/feat/pending-work-2026-09

feat: land pending September work
This commit is contained in:
dualfroz authored and GitHub committed 2026-09-14 03:26:40 +02:00
commit 4e69d5c6ce
36 files changed
+7849 -74

No files matched your search

+14 -1
View File
@@ -11,8 +11,12 @@ defmodule Excache.Application do
{Registry, keys: :duplicate, name: Excache.PubSub.Registry},
Excache.Store.Supervisor,
Excache.Aof,
Excache.Blocking,
Excache.Expiry,
{DynamicSupervisor, strategy: :one_for_one, name: Excache.Server.ConnectionSupervisor}
trackerSpec(),
Excache.Eviction.Runner,
{DynamicSupervisor, strategy: :one_for_one, name: Excache.Server.ConnectionSupervisor},
Excache.Clients
]
case Supervisor.start_link(children, strategy: :one_for_one, name: Excache.Supervisor) do
@@ -25,6 +29,15 @@ defmodule Excache.Application do
end
end
def trackerSpec do
{Excache.Eviction.Tracker,
[
name: Excache.Eviction.Tracker,
logFactor: Excache.Config.lfuLogFactor(),
decayTime: Excache.Config.lfuDecayTime()
]}
end
defp start_acceptor(supervisor, port) when is_integer(port) do
case Supervisor.start_child(supervisor, {Excache.Server.Acceptor, [port: port]}) do
{:ok, _pid} -> {:ok, supervisor}
+280
View File
@@ -0,0 +1,280 @@
defmodule Excache.Blocking do
@moduledoc false
use GenServer
alias Excache.Store
alias Excache.Command.Reply
@name __MODULE__
def start_link(_arg), do: GenServer.start_link(__MODULE__, nil, name: @name)
def acquire(request), do: GenServer.call(@name, {:acquire, request, :block})
def attempt(request) do
{:served, term} = GenServer.call(@name, {:acquire, request, :immediate})
term
end
def signal([]), do: []
def signal(keys), do: GenServer.call(@name, {:ready, keys})
def takeWrites, do: GenServer.call(@name, :takeWrites)
def waitingCount(key), do: GenServer.call(@name, {:waitingCount, key})
@impl true
def init(_arg), do: {:ok, %{queues: %{}, waiters: %{}, monitors: %{}, writes: []}}
@impl true
def handle_call({:acquire, request, mode}, {pid, _tag}, state) do
waiter = %{
ref: make_ref(),
pid: pid,
keys: request.keys,
op: request.op,
monitor: nil,
timer: nil
}
state = enqueue(state, waiter)
{state, served} = drain(state, request.keys, [])
case {deliver(served, waiter.ref), mode} do
{nil, :immediate} -> {:reply, {:served, missReply(request.op)}, dropWaiter(state, waiter)}
{nil, :block} -> {:reply, {:blocked, waiter.ref}, arm(state, waiter, request.timeout)}
{term, _mode} -> {:reply, {:served, term}, state}
end
end
def handle_call({:ready, keys}, _from, state) do
{state, served} = drain(state, keys, [])
deliver(served, nil)
takeWrites(state)
end
def handle_call(:takeWrites, _from, state), do: takeWrites(state)
def handle_call({:waitingCount, key}, _from, state) do
count =
case Map.fetch(state.queues, key) do
{:ok, queue} -> :queue.len(queue)
:error -> 0
end
{:reply, count, state}
end
@impl true
def handle_info({:blockingTimeout, ref}, state) do
case Map.fetch(state.waiters, ref) do
{:ok, waiter} ->
send(waiter.pid, {:excacheUnblocked, ref, {:array, nil}})
{:noreply, dropWaiter(state, waiter)}
:error ->
{:noreply, state}
end
end
def handle_info({:DOWN, monitor, :process, _pid, _reason}, state) do
ref = Map.fetch!(state.monitors, monitor)
{:noreply, dropWaiter(state, Map.fetch!(state.waiters, ref))}
end
defp drain(state, [], served), do: {state, served}
defp drain(state, [key | rest], served) do
{state, served, chained} = drainKey(state, key, served, [])
drain(state, chained ++ rest, served)
end
defp drainKey(state, key, served, chained) do
case headWaiter(state, key) do
nil ->
{state, served, chained}
waiter ->
case serveWaiter(waiter) do
:empty ->
{state, served, chained}
{:ok, term, ready, writes} ->
state = recordWrites(dropWaiter(state, waiter), writes)
drainKey(state, key, [{waiter, term} | served], ready ++ chained)
{:error, term} ->
drainKey(dropWaiter(state, waiter), key, [{waiter, term} | served], chained)
end
end
end
defp recordWrites(state, writes), do: %{state | writes: state.writes ++ writes}
defp takeWrites(state), do: {:reply, state.writes, %{state | writes: []}}
defp serveWaiter(waiter) do
Enum.reduce_while(waiter.keys, :empty, fn key, _pending ->
case serve(waiter.op, key) do
:empty -> {:cont, :empty}
outcome -> {:halt, outcome}
end
end)
end
defp serve({:pop, side}, key) do
case popOne(key, side) do
{:ok, value} ->
{:ok, {:array, [{:bulk, key}, {:bulk, value}]}, [], [{popName(side), [key]}]}
nil ->
:empty
{:error, reason} ->
{:error, errorTerm(reason)}
end
end
defp serve({:mpop, side, count}, key) do
case popMany(key, side, count) do
{:ok, [_ | _] = values} ->
taken = Integer.to_string(length(values))
{:ok, {:array, [{:bulk, key}, {:array, Reply.bulks(values)}]}, [],
[{popName(side), [key, taken]}]}
{:ok, []} ->
:empty
nil ->
:empty
{:error, reason} ->
{:error, errorTerm(reason)}
end
end
defp serve({:move, fromSide, toSide, destination}, key) do
case popOne(key, fromSide) do
{:ok, value} ->
moveInto(key, destination, fromSide, toSide, value)
nil ->
:empty
{:error, reason} ->
{:error, errorTerm(reason)}
end
end
defp moveInto(source, destination, fromSide, toSide, value) do
case pushOne(destination, toSide, value) do
{:ok, _length} ->
{:ok, {:bulk, value}, [destination],
moveWrites(source, destination, fromSide, toSide, value)}
{:error, reason} ->
pushOne(source, fromSide, value)
{:error, errorTerm(reason)}
end
end
defp popOne(key, :left), do: Store.lpop(key, nil)
defp popOne(key, :right), do: Store.rpop(key, nil)
defp popMany(key, :left, count), do: Store.lpop(key, count)
defp popMany(key, :right, count), do: Store.rpop(key, count)
defp pushOne(key, :left, value), do: Store.lpush(key, [value])
defp pushOne(key, :right, value), do: Store.rpush(key, [value])
defp popName(:left), do: "LPOP"
defp popName(:right), do: "RPOP"
defp pushName(:left), do: "LPUSH"
defp pushName(:right), do: "RPUSH"
defp moveWrites(source, destination, :right, :left, _value),
do: [{"RPOPLPUSH", [source, destination]}]
defp moveWrites(source, destination, fromSide, toSide, value),
do: [{popName(fromSide), [source]}, {pushName(toSide), [destination, value]}]
defp missReply({:move, _fromSide, _toSide, _destination}), do: {:bulk, nil}
defp missReply(_op), do: {:array, nil}
defp errorTerm(:wrongtype), do: Reply.wrongtype()
defp deliver(served, inlineRef) do
Enum.reduce(served, nil, fn {waiter, term}, inline ->
if waiter.ref == inlineRef do
term
else
send(waiter.pid, {:excacheUnblocked, waiter.ref, term})
inline
end
end)
end
defp headWaiter(state, key) do
with {:ok, queue} <- Map.fetch(state.queues, key),
{:value, ref} <- :queue.peek(queue) do
Map.fetch!(state.waiters, ref)
else
_missing -> nil
end
end
defp enqueue(state, waiter) do
queues =
Enum.reduce(Enum.uniq(waiter.keys), state.queues, fn key, acc ->
Map.put(acc, key, :queue.in(waiter.ref, Map.get(acc, key, :queue.new())))
end)
%{state | queues: queues, waiters: Map.put(state.waiters, waiter.ref, waiter)}
end
defp dropWaiter(state, waiter) do
armed = Map.fetch!(state.waiters, waiter.ref)
if armed.monitor, do: Process.demonitor(armed.monitor, [:flush])
if armed.timer, do: Process.cancel_timer(armed.timer)
queues =
Enum.reduce(Enum.uniq(armed.keys), state.queues, fn key, acc ->
dropFromQueue(acc, key, armed.ref)
end)
%{
state
| queues: queues,
waiters: Map.delete(state.waiters, armed.ref),
monitors: Map.delete(state.monitors, armed.monitor)
}
end
defp dropFromQueue(queues, key, ref) do
pruned = :queue.filter(&(&1 != ref), Map.fetch!(queues, key))
if :queue.is_empty(pruned) do
Map.delete(queues, key)
else
Map.put(queues, key, pruned)
end
end
defp arm(state, waiter, timeout) do
monitor = Process.monitor(waiter.pid)
timer =
if timeout > 0, do: Process.send_after(self(), {:blockingTimeout, waiter.ref}, timeout)
armed = %{waiter | monitor: monitor, timer: timer}
%{
state
| waiters: Map.put(state.waiters, waiter.ref, armed),
monitors: Map.put(state.monitors, monitor, waiter.ref)
}
end
end
+239
View File
@@ -0,0 +1,239 @@
defmodule Excache.Clients do
@moduledoc false
use GenServer
@name __MODULE__
@supervisor Excache.Server.ConnectionSupervisor
@pubsubRegistry Excache.PubSub.Registry
@defaultPoll 500
@selfKillDelay 50
def start_link(_arg), do: GenServer.start_link(__MODULE__, nil, name: @name)
def snapshot, do: GenServer.call(@name, :snapshot)
def find(pid), do: GenServer.call(@name, {:find, pid})
def setName(pid, name), do: GenServer.call(@name, {:setName, pid, name})
def kill(matcher, caller), do: GenServer.call(@name, {:kill, matcher, caller})
def subscriptionCount(pid), do: length(Registry.keys(@pubsubRegistry, pid))
def subscribedChannels do
@pubsubRegistry
|> Registry.select([{{:"$1", :_, :_}, [], [:"$1"]}])
|> Enum.uniq()
|> length()
end
def blocked?(pid) do
case Process.info(pid, :monitored_by) do
{:monitored_by, watchers} -> Process.whereis(Excache.Blocking) in watchers
nil -> false
end
end
@impl true
def init(_arg) do
schedule()
{:ok,
%{
nextId: 1,
clients: %{},
monitors: %{},
connectionsReceived: 0,
closedNetIn: 0,
closedNetOut: 0,
runId: newRunId(),
startedAt: System.system_time(:millisecond)
}}
end
@impl true
def handle_call(:snapshot, _from, state) do
state = sync(state)
{:reply, buildSnapshot(state), state}
end
@impl true
def handle_call({:find, pid}, _from, state) do
state = sync(state)
{:reply, Map.get(state.clients, pid), state}
end
@impl true
def handle_call({:setName, pid, name}, _from, state) do
state = sync(state)
case Map.fetch(state.clients, pid) do
{:ok, client} ->
renamed = Map.put(state.clients, pid, %{client | name: name})
{:reply, :ok, %{state | clients: renamed}}
:error ->
{:reply, :error, state}
end
end
@impl true
def handle_call({:kill, matcher, caller}, _from, state) do
state = sync(state)
victims = state.clients |> Map.values() |> Enum.filter(matcher)
{itself, others} = Enum.split_with(victims, &(&1.pid == caller))
Enum.each(others, fn client -> terminate(client.pid) end)
Enum.each(itself, fn client ->
Process.send_after(self(), {:terminate, client.pid}, @selfKillDelay)
end)
{:reply, length(victims), state}
end
@impl true
def handle_info(:poll, state) do
schedule()
{:noreply, sync(state)}
end
@impl true
def handle_info({:terminate, pid}, state) do
terminate(pid)
{:noreply, state}
end
@impl true
def handle_info({:DOWN, monitor, :process, pid, _reason}, state) do
{_pid, monitors} = Map.pop(state.monitors, monitor)
{client, clients} = Map.pop(state.clients, pid)
{:noreply, accountClosed(%{state | monitors: monitors, clients: clients}, client)}
end
defp schedule do
interval = Application.get_env(:excache, :clientsPollInterval, @defaultPoll)
Process.send_after(self(), :poll, interval)
end
defp sync(state) do
Enum.reduce(livePids(), state, fn pid, acc ->
case Map.fetch(acc.clients, pid) do
{:ok, client} -> %{acc | clients: Map.put(acc.clients, pid, refresh(client))}
:error -> register(acc, pid)
end
end)
end
defp livePids do
for {_id, pid, _type, _modules} <- DynamicSupervisor.which_children(@supervisor),
is_pid(pid),
do: pid
end
defp register(state, pid) do
with socket when socket != nil <- socketOf(pid),
{netIn, netOut} <- netStats(socket) do
now = System.system_time(:millisecond)
client = %{
id: state.nextId,
pid: pid,
socket: socket,
addr: addressOf(socket, &:inet.peername/1),
laddr: addressOf(socket, &:inet.sockname/1),
name: "",
connectedAt: now,
lastActiveAt: now,
netIn: netIn,
netOut: netOut
}
%{
state
| nextId: state.nextId + 1,
clients: Map.put(state.clients, pid, client),
monitors: Map.put(state.monitors, Process.monitor(pid), pid),
connectionsReceived: state.connectionsReceived + 1
}
else
_unavailable -> state
end
end
defp refresh(client) do
case netStats(client.socket) do
{netIn, netOut} when netIn > client.netIn ->
%{
client
| netIn: netIn,
netOut: netOut,
lastActiveAt: System.system_time(:millisecond)
}
{netIn, netOut} ->
%{client | netIn: netIn, netOut: netOut}
:closed ->
client
end
end
defp accountClosed(state, nil), do: state
defp accountClosed(state, client) do
%{
state
| closedNetIn: state.closedNetIn + client.netIn,
closedNetOut: state.closedNetOut + client.netOut
}
end
defp buildSnapshot(state) do
clients = state.clients |> Map.values() |> Enum.sort_by(& &1.id)
%{
clients: clients,
connectionsReceived: state.connectionsReceived,
netIn: Enum.reduce(clients, state.closedNetIn, fn client, sum -> sum + client.netIn end),
netOut: Enum.reduce(clients, state.closedNetOut, fn client, sum -> sum + client.netOut end),
runId: state.runId,
startedAt: state.startedAt
}
end
defp terminate(pid), do: DynamicSupervisor.terminate_child(@supervisor, pid)
defp socketOf(pid) do
case Process.info(pid, :links) do
{:links, links} -> Enum.find(links, &tcpPort?/1)
nil -> nil
end
end
defp tcpPort?(link) when is_port(link) do
match?({:name, ~c"tcp_inet"}, Port.info(link, :name))
end
defp tcpPort?(_link), do: false
defp netStats(socket) do
case :inet.getstat(socket, [:recv_oct, :send_oct]) do
{:ok, [recv_oct: netIn, send_oct: netOut]} -> {netIn, netOut}
{:error, _reason} -> :closed
end
end
defp addressOf(socket, fetch) do
case fetch.(socket) do
{:ok, {ip, port}} -> formatAddress(ip, port)
{:error, _reason} -> ""
end
end
defp formatAddress(ip, port) when tuple_size(ip) == 8, do: "[#{:inet.ntoa(ip)}]:#{port}"
defp formatAddress(ip, port), do: "#{:inet.ntoa(ip)}:#{port}"
defp newRunId, do: 20 |> :rand.bytes() |> Base.encode16(case: :lower)
end
+79 -12
View File
@@ -1,8 +1,9 @@
defmodule Excache.Command do
@moduledoc false
alias Excache.{PubSub, Store, Persistence, Aof, Notify}
alias Excache.{Blocking, PubSub, Store, Persistence, Aof, Notify}
alias Excache.Command.{Reply, Transaction}
alias Excache.Eviction.Runner
import Reply,
only: [
@@ -45,10 +46,25 @@ defmodule Excache.Command do
ZADD ZREM ZINCRBY ZPOPMIN ZPOPMAX
)
@oomDenied ~w(
SET SETNX SETRANGE APPEND GETSET MSET INCR INCRBY DECR DECRBY INCRBYFLOAT
LPUSH RPUSH LINSERT LSET RPOPLPUSH BLMOVE BRPOPLPUSH
HSET HSETNX HINCRBY
SADD SMOVE SINTERSTORE SUNIONSTORE SDIFFSTORE
ZADD ZINCRBY
XADD XSETID XGROUP XREADGROUP XCLAIM XAUTOCLAIM
SETBIT BITOP BITFIELD PFADD PFMERGE
COPY RENAME RENAMENX
)
@oom {:error, "OOM command not allowed when used memory > 'maxmemory'"}
def known_commands, do: @known ++ Excache.Command.Extended.command_names()
@extended_write Excache.Command.Extended.write_command_names()
@blockingNames Excache.Command.Blocking.commandNames()
def execute([], state), do: {[], state}
def execute([name | args], state) do
@@ -61,27 +77,76 @@ defmodule Excache.Command do
end
def dispatch_and_log(command, args, state) do
{replies, updated} = dispatch(command, args, state)
if writeCommand?(command) and not Enum.any?(replies, &match?({:error, _}, &1)) do
Aof.append(command, aofArgs(command, args, replies))
Notify.for_command(command, args)
if oomDenied?(command) and Runner.admit() == :oom do
{[@oom], state}
else
logged(command, args, state)
end
end
defp logged(command, args, state) do
{replies, updated} = dispatch(command, args, state)
succeeded = not Enum.any?(replies, &match?({:error, _}, &1))
if succeeded and oomDenied?(command), do: Runner.account(args)
entries = if succeeded, do: aofEntries(command, args, replies), else: []
logWrites(entries)
logWrites(servedWrites(command, wakeKeys(entries)))
{replies, updated}
end
defp logWrites(entries) do
Enum.each(entries, fn {name, args} ->
if writeCommand?(name) do
Aof.append(name, args)
Notify.for_command(name, args)
end
end)
end
defp servedWrites(command, keys) do
if command in @blockingNames, do: Blocking.takeWrites(), else: Blocking.signal(keys)
end
defp wakeKeys(entries), do: Enum.flat_map(entries, &listTargets/1)
defp listTargets({"LPUSH", [key | _values]}), do: [key]
defp listTargets({"RPUSH", [key | _values]}), do: [key]
defp listTargets({"LINSERT", [key | _rest]}), do: [key]
defp listTargets({"RPOPLPUSH", [_source, destination]}), do: [destination]
defp listTargets({"RENAME", [_source, destination]}), do: [destination]
defp listTargets({"RENAMENX", [_source, destination]}), do: [destination]
defp listTargets({"COPY", [_source, destination | _options]}), do: [destination]
defp listTargets(_entry), do: []
defp writeCommand?(command), do: command in @write_commands or command in @extended_write
defp aofArgs("XADD", args, [{:bulk, id}]) when is_binary(id),
do: Excache.Command.Streams.rewriteAddId(args, id)
defp oomDenied?(command), do: command in @oomDenied
defp aofArgs(_command, args, _replies), do: args
defp aofEntries("XADD", args, [{:bulk, id}]) when is_binary(id),
do: [{"XADD", Excache.Command.Streams.rewriteAddId(args, id)}]
defp aofEntries("SPOP", [key | _rest] = args, replies) do
case poppedMembers(replies) do
[] -> [{"SPOP", args}]
members -> [{"SREM", [key | members]}]
end
end
defp aofEntries(command, args, _replies), do: [{command, args}]
defp poppedMembers([{:bulk, member}]) when is_binary(member), do: [member]
defp poppedMembers([{:array, entries}]),
do: for({:bulk, member} <- entries, is_binary(member), do: member)
defp poppedMembers(_replies), do: []
defp dispatch("PING", [], state), do: {[{:simple, "PONG"}], state}
defp dispatch("PING", [message], state), do: {[{:bulk, message}], state}
defp dispatch("ECHO", [message], state), do: {[{:bulk, message}], state}
defp dispatch("COMMAND", _args, state), do: {[{:array, []}], state}
defp dispatch("SELECT", [_db], state), do: {[{:simple, "OK"}], state}
defp dispatch("QUIT", [], state), do: {[{:simple, "OK"}], state}
defp dispatch("DBSIZE", [], state), do: {[{:integer, Store.dbsize()}], state}
@@ -287,13 +352,15 @@ defmodule Excache.Command do
end
end
defp dispatch("ZPOPMIN", [key], state), do: reply(zrange_reply(Store.zpopmin(key, 1), true), state)
defp dispatch("ZPOPMIN", [key], state),
do: reply(zrange_reply(Store.zpopmin(key, 1), true), state)
defp dispatch("ZPOPMIN", [key, count], state) do
with_int(count, state, fn n -> zrange_reply(Store.zpopmin(key, n), true) end)
end
defp dispatch("ZPOPMAX", [key], state), do: reply(zrange_reply(Store.zpopmax(key, 1), true), state)
defp dispatch("ZPOPMAX", [key], state),
do: reply(zrange_reply(Store.zpopmax(key, 1), true), state)
defp dispatch("ZPOPMAX", [key, count], state) do
with_int(count, state, fn n -> zrange_reply(Store.zpopmax(key, n), true) end)
+119
View File
@@ -0,0 +1,119 @@
defmodule Excache.Command.Blocking do
@moduledoc false
alias Excache.Command.Reply
@names ~w(BLPOP BRPOP BLMOVE BRPOPLPUSH BLMPOP)
def commandNames, do: @names
def dispatch("BLPOP", args, state), do: popCommand(args, :left, state)
def dispatch("BRPOP", args, state), do: popCommand(args, :right, state)
def dispatch("BLMOVE", [source, destination, fromRaw, toRaw, timeoutRaw], state) do
with {:ok, fromSide} <- parseSide(fromRaw),
{:ok, toSide} <- parseSide(toRaw) do
withTimeout(timeoutRaw, state, fn timeout ->
submit([source], {:move, fromSide, toSide, destination}, timeout, state)
end)
else
:error -> {[Reply.syntax()], state}
end
end
def dispatch("BRPOPLPUSH", [source, destination, timeoutRaw], state) do
withTimeout(timeoutRaw, state, fn timeout ->
submit([source], {:move, :right, :left, destination}, timeout, state)
end)
end
def dispatch("BLMPOP", [timeoutRaw, numkeysRaw | rest], state) when rest != [] do
withTimeout(timeoutRaw, state, fn timeout ->
case parseMpop(numkeysRaw, rest) do
{:ok, keys, side, count} -> submit(keys, {:mpop, side, count}, timeout, state)
{:reply, term} -> {[term], state}
end
end)
end
def dispatch(_name, _args, _state), do: :unhandled
defp popCommand(args, side, state) when length(args) >= 2 do
{keys, [timeoutRaw]} = Enum.split(args, length(args) - 1)
withTimeout(timeoutRaw, state, fn timeout ->
submit(keys, {:pop, side}, timeout, state)
end)
end
defp popCommand(_args, _side, _state), do: :unhandled
defp submit(keys, op, timeout, state) do
request = %{keys: keys, op: op, timeout: timeout}
if state.in_multi do
{[Excache.Blocking.attempt(request)], state}
else
case Excache.Blocking.acquire(request) do
{:served, term} -> {[term], state}
{:blocked, ref} -> {[{:blocked, ref}], state}
end
end
end
defp withTimeout(raw, state, fun) do
case parseTimeout(raw) do
{:ok, timeout} -> fun.(timeout)
{:reply, term} -> {[term], state}
end
end
defp parseTimeout(raw) do
case Reply.parse_score(raw) do
{:ok, seconds} when seconds < 0 -> {:reply, {:error, "ERR timeout is negative"}}
{:ok, seconds} -> {:ok, trunc(seconds * 1000)}
:error -> {:reply, {:error, "ERR timeout is not a float or out of range"}}
end
end
defp parseSide(raw) do
case String.upcase(raw) do
"LEFT" -> {:ok, :left}
"RIGHT" -> {:ok, :right}
_other -> :error
end
end
defp parseMpop(numkeysRaw, rest) do
case Integer.parse(numkeysRaw) do
{numkeys, ""} when numkeys > 0 -> splitMpop(numkeys, rest)
_invalid -> {:reply, {:error, "ERR numkeys should be greater than 0"}}
end
end
defp splitMpop(numkeys, rest) when length(rest) > numkeys do
{keys, [directionRaw | tail]} = Enum.split(rest, numkeys)
case parseSide(directionRaw) do
{:ok, side} -> mpopCount(keys, side, tail)
:error -> {:reply, Reply.syntax()}
end
end
defp splitMpop(_numkeys, _rest), do: {:reply, Reply.syntax()}
defp mpopCount(keys, side, []), do: {:ok, keys, side, 1}
defp mpopCount(keys, side, [countToken, countRaw]) do
if String.upcase(countToken) == "COUNT" do
case Integer.parse(countRaw) do
{count, ""} when count > 0 -> {:ok, keys, side, count}
_invalid -> {:reply, {:error, "ERR count should be greater than 0"}}
end
else
{:reply, Reply.syntax()}
end
end
defp mpopCount(_keys, _side, _tail), do: {:reply, Reply.syntax()}
end
+12 -2
View File
@@ -1,7 +1,7 @@
defmodule Excache.Command.Extended do
@moduledoc false
alias Excache.Command.{Streams, Bitmaps, HyperLogLog, Keyspace}
alias Excache.Command.{Streams, Bitmaps, HyperLogLog, Keyspace, Server, Blocking, Introspection}
@stream ~w(XADD XLEN XRANGE XREVRANGE XDEL XTRIM XSETID XREAD XINFO XGROUP
XREADGROUP XACK XPENDING XCLAIM XAUTOCLAIM)
@@ -17,17 +17,27 @@ defmodule Excache.Command.Extended do
ZCOUNT ZREVRANGE ZREVRANK ZMSCORE
RENAME RENAMENX COPY TOUCH UNLINK EXPIREAT EXPIRETIME PEXPIRETIME)
@server ~w(MEMORY CONFIG)
@blocking Excache.Command.Blocking.commandNames()
@introspection Excache.Command.Introspection.commandNames()
def dispatch(name, args, state) do
cond do
name in @stream -> Streams.dispatch(name, args, state)
name in @bitmap -> Bitmaps.dispatch(name, args, state)
name in @hll -> HyperLogLog.dispatch(name, args, state)
name in @keyspace -> Keyspace.dispatch(name, args, state)
name in @server -> Server.dispatch(name, args, state)
name in @blocking -> Blocking.dispatch(name, args, state)
name in @introspection -> Introspection.dispatch(name, args, state)
true -> :unhandled
end
end
def command_names, do: @stream ++ @bitmap ++ @hll ++ @keyspace
def command_names,
do: @stream ++ @bitmap ++ @hll ++ @keyspace ++ @server ++ @blocking ++ @introspection
def write_command_names do
~w(XADD XDEL XTRIM XSETID XGROUP XREADGROUP XACK XCLAIM XAUTOCLAIM
+91
View File
@@ -0,0 +1,91 @@
defmodule Excache.Command.Introspection do
@moduledoc false
alias Excache.Command.Introspection.Catalog
alias Excache.Command.Introspection.Client
alias Excache.Command.Introspection.Info
@names ~w(INFO CLIENT COMMAND)
def commandNames, do: @names
def dispatch("INFO", args, state), do: {[{:bulk, Info.render(args)}], state}
def dispatch("CLIENT", [subcommand | rest], state),
do: Client.dispatch(String.upcase(subcommand), rest, state)
def dispatch("CLIENT", [], state), do: {[wrongArity("client")], state}
def dispatch("COMMAND", [], state), do: {[fullTable()], state}
def dispatch("COMMAND", [subcommand | rest], state),
do: command(String.upcase(subcommand), rest, state)
defp command("COUNT", [], state), do: {[{:integer, Catalog.count()}], state}
defp command("LIST", [], state) do
{[{:array, Enum.map(Catalog.names(), fn name -> {:bulk, String.downcase(name)} end)}], state}
end
defp command("INFO", [], state), do: {[fullTable()], state}
defp command("INFO", names, state) do
{[{:array, Enum.map(names, fn name -> Catalog.infoTerm(String.upcase(name)) end)}], state}
end
defp command("DOCS", [], state) do
{[{:array, Enum.flat_map(Catalog.names(), &docsEntry/1)}], state}
end
defp command("DOCS", names, state) do
entries =
names
|> Enum.map(&String.upcase/1)
|> Enum.filter(&Catalog.known?/1)
|> Enum.flat_map(&docsEntry/1)
{[{:array, entries}], state}
end
defp command("GETKEYS", [name | args], state) do
{[getkeys(String.upcase(name), args)], state}
end
defp command("GETKEYS", [], state), do: {[wrongArity("command|getkeys")], state}
defp command(subcommand, _args, state) do
{[
{:error,
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
"Try COMMAND HELP."}
], state}
end
defp fullTable, do: {:array, Enum.map(Catalog.names(), &Catalog.infoTerm/1)}
defp docsEntry(name), do: [{:bulk, String.downcase(name)}, Catalog.docsTerm(name)]
defp getkeys(name, args) do
cond do
not Catalog.known?(name) ->
{:error, "ERR Invalid command specified"}
not Catalog.arityOk?(name, length(args) + 1) ->
{:error, "ERR Invalid number of arguments specified for command"}
true ->
extractKeys(name, args)
end
end
defp extractKeys(name, args) do
case Catalog.keysFor(name, args) do
{:ok, []} -> {:error, "ERR The command has no key arguments"}
{:ok, keys} -> {:array, Enum.map(keys, fn key -> {:bulk, key} end)}
:error -> {:error, "ERR Invalid arguments specified for command"}
end
end
defp wrongArity(name),
do: {:error, "ERR wrong number of arguments for '#{name}' command"}
end
@@ -0,0 +1,259 @@
defmodule Excache.Command.Introspection.Catalog do
@moduledoc false
# {arity, flags, firstKey, lastKey, step, group}
# Arity counts the command name, negative means "at least this many".
# firstKey/lastKey/step are 1 based positions in the full argument vector,
# a negative lastKey counts back from the end, zeroes mean "takes no keys".
@commands %{
"PING" => {-1, [:fast], 0, 0, 0, :connection},
"ECHO" => {2, [:fast], 0, 0, 0, :connection},
"QUIT" => {-1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :connection},
"SELECT" => {2, [:loading, :stale, :fast], 0, 0, 0, :connection},
"CLIENT" => {-2, [:admin, :noscript, :loading, :stale], 0, 0, 0, :connection},
"COMMAND" => {-1, [:loading, :stale], 0, 0, 0, :server},
"INFO" => {-1, [:loading, :stale], 0, 0, 0, :server},
"CONFIG" => {-2, [:admin, :noscript, :loading, :stale], 0, 0, 0, :server},
"MEMORY" => {-2, [:readonly], 0, 0, 0, :server},
"DBSIZE" => {1, [:readonly, :fast], 0, 0, 0, :server},
"FLUSHDB" => {-1, [:write], 0, 0, 0, :server},
"SAVE" => {1, [:admin, :noscript], 0, 0, 0, :server},
"BGSAVE" => {-1, [:admin, :noscript], 0, 0, 0, :server},
"GET" => {2, [:readonly, :fast], 1, 1, 1, :string},
"SET" => {-3, [:write, :denyoom], 1, 1, 1, :string},
"SETNX" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
"GETSET" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
"GETDEL" => {2, [:write, :fast], 1, 1, 1, :string},
"APPEND" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
"STRLEN" => {2, [:readonly, :fast], 1, 1, 1, :string},
"SETRANGE" => {4, [:write, :denyoom], 1, 1, 1, :string},
"GETRANGE" => {4, [:readonly], 1, 1, 1, :string},
"INCR" => {2, [:write, :denyoom, :fast], 1, 1, 1, :string},
"DECR" => {2, [:write, :denyoom, :fast], 1, 1, 1, :string},
"INCRBY" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
"DECRBY" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
"INCRBYFLOAT" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
"MSET" => {-3, [:write, :denyoom], 1, -1, 2, :string},
"MGET" => {-2, [:readonly, :fast], 1, -1, 1, :string},
"DEL" => {-2, [:write], 1, -1, 1, :generic},
"UNLINK" => {-2, [:write, :fast], 1, -1, 1, :generic},
"EXISTS" => {-2, [:readonly, :fast], 1, -1, 1, :generic},
"TOUCH" => {-2, [:readonly, :fast], 1, -1, 1, :generic},
"KEYS" => {2, [:readonly], 0, 0, 0, :generic},
"SCAN" => {-2, [:readonly], 0, 0, 0, :generic},
"TYPE" => {2, [:readonly, :fast], 1, 1, 1, :generic},
"OBJECT" => {-2, [:readonly], 0, 0, 0, :generic},
"RENAME" => {3, [:write], 1, 2, 1, :generic},
"RENAMENX" => {3, [:write, :fast], 1, 2, 1, :generic},
"COPY" => {-3, [:write, :denyoom], 1, 2, 1, :generic},
"EXPIRE" => {-3, [:write, :fast], 1, 1, 1, :generic},
"PEXPIRE" => {-3, [:write, :fast], 1, 1, 1, :generic},
"EXPIREAT" => {-3, [:write, :fast], 1, 1, 1, :generic},
"EXPIRETIME" => {2, [:readonly, :fast], 1, 1, 1, :generic},
"PEXPIRETIME" => {2, [:readonly, :fast], 1, 1, 1, :generic},
"TTL" => {2, [:readonly, :fast], 1, 1, 1, :generic},
"PTTL" => {2, [:readonly, :fast], 1, 1, 1, :generic},
"PERSIST" => {2, [:write, :fast], 1, 1, 1, :generic},
"LPUSH" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :list},
"RPUSH" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :list},
"LPOP" => {-2, [:write, :fast], 1, 1, 1, :list},
"RPOP" => {-2, [:write, :fast], 1, 1, 1, :list},
"LRANGE" => {4, [:readonly], 1, 1, 1, :list},
"LLEN" => {2, [:readonly, :fast], 1, 1, 1, :list},
"LINDEX" => {3, [:readonly], 1, 1, 1, :list},
"LSET" => {4, [:write, :denyoom], 1, 1, 1, :list},
"LREM" => {4, [:write], 1, 1, 1, :list},
"LTRIM" => {4, [:write], 1, 1, 1, :list},
"LINSERT" => {5, [:write, :denyoom], 1, 1, 1, :list},
"LPOS" => {-3, [:readonly], 1, 1, 1, :list},
"RPOPLPUSH" => {3, [:write, :denyoom], 1, 2, 1, :list},
"BLPOP" => {-3, [:write, :noscript, :blocking], 1, -2, 1, :list},
"BRPOP" => {-3, [:write, :noscript, :blocking], 1, -2, 1, :list},
"BLMOVE" => {6, [:write, :denyoom, :noscript, :blocking], 1, 2, 1, :list},
"BRPOPLPUSH" => {4, [:write, :denyoom, :noscript, :blocking], 1, 2, 1, :list},
"BLMPOP" => {-5, [:write, :noscript, :blocking, :movablekeys], 0, 0, 0, :list},
"HSET" => {-4, [:write, :denyoom, :fast], 1, 1, 1, :hash},
"HSETNX" => {4, [:write, :denyoom, :fast], 1, 1, 1, :hash},
"HGET" => {3, [:readonly, :fast], 1, 1, 1, :hash},
"HMGET" => {-3, [:readonly, :fast], 1, 1, 1, :hash},
"HDEL" => {-3, [:write, :fast], 1, 1, 1, :hash},
"HGETALL" => {2, [:readonly], 1, 1, 1, :hash},
"HKEYS" => {2, [:readonly], 1, 1, 1, :hash},
"HVALS" => {2, [:readonly], 1, 1, 1, :hash},
"HLEN" => {2, [:readonly, :fast], 1, 1, 1, :hash},
"HEXISTS" => {3, [:readonly, :fast], 1, 1, 1, :hash},
"HINCRBY" => {4, [:write, :denyoom, :fast], 1, 1, 1, :hash},
"HSCAN" => {-3, [:readonly], 1, 1, 1, :hash},
"SADD" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :set},
"SREM" => {-3, [:write, :fast], 1, 1, 1, :set},
"SMEMBERS" => {2, [:readonly], 1, 1, 1, :set},
"SISMEMBER" => {3, [:readonly, :fast], 1, 1, 1, :set},
"SMISMEMBER" => {-3, [:readonly, :fast], 1, 1, 1, :set},
"SCARD" => {2, [:readonly, :fast], 1, 1, 1, :set},
"SPOP" => {-2, [:write, :fast], 1, 1, 1, :set},
"SRANDMEMBER" => {-2, [:readonly], 1, 1, 1, :set},
"SMOVE" => {4, [:write, :fast], 1, 2, 1, :set},
"SINTER" => {-2, [:readonly], 1, -1, 1, :set},
"SUNION" => {-2, [:readonly], 1, -1, 1, :set},
"SDIFF" => {-2, [:readonly], 1, -1, 1, :set},
"SINTERCARD" => {-3, [:readonly, :movablekeys], 0, 0, 0, :set},
"SINTERSTORE" => {-3, [:write, :denyoom], 1, -1, 1, :set},
"SUNIONSTORE" => {-3, [:write, :denyoom], 1, -1, 1, :set},
"SDIFFSTORE" => {-3, [:write, :denyoom], 1, -1, 1, :set},
"SSCAN" => {-3, [:readonly], 1, 1, 1, :set},
"ZADD" => {-4, [:write, :denyoom, :fast], 1, 1, 1, :zset},
"ZINCRBY" => {4, [:write, :denyoom, :fast], 1, 1, 1, :zset},
"ZREM" => {-3, [:write, :fast], 1, 1, 1, :zset},
"ZSCORE" => {3, [:readonly, :fast], 1, 1, 1, :zset},
"ZMSCORE" => {-3, [:readonly, :fast], 1, 1, 1, :zset},
"ZRANK" => {3, [:readonly, :fast], 1, 1, 1, :zset},
"ZREVRANK" => {3, [:readonly, :fast], 1, 1, 1, :zset},
"ZCARD" => {2, [:readonly, :fast], 1, 1, 1, :zset},
"ZCOUNT" => {4, [:readonly, :fast], 1, 1, 1, :zset},
"ZRANGE" => {-4, [:readonly], 1, 1, 1, :zset},
"ZREVRANGE" => {-4, [:readonly], 1, 1, 1, :zset},
"ZRANGEBYSCORE" => {-4, [:readonly], 1, 1, 1, :zset},
"ZPOPMIN" => {-2, [:write, :fast], 1, 1, 1, :zset},
"ZPOPMAX" => {-2, [:write, :fast], 1, 1, 1, :zset},
"ZSCAN" => {-3, [:readonly], 1, 1, 1, :zset},
"XADD" => {-5, [:write, :denyoom, :fast], 1, 1, 1, :stream},
"XLEN" => {2, [:readonly, :fast], 1, 1, 1, :stream},
"XRANGE" => {-4, [:readonly], 1, 1, 1, :stream},
"XREVRANGE" => {-4, [:readonly], 1, 1, 1, :stream},
"XDEL" => {-3, [:write, :fast], 1, 1, 1, :stream},
"XTRIM" => {-4, [:write], 1, 1, 1, :stream},
"XSETID" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :stream},
"XACK" => {-4, [:write, :fast], 1, 1, 1, :stream},
"XPENDING" => {-3, [:readonly], 1, 1, 1, :stream},
"XCLAIM" => {-6, [:write, :fast], 1, 1, 1, :stream},
"XAUTOCLAIM" => {-7, [:write, :fast], 1, 1, 1, :stream},
"XREAD" => {-4, [:readonly, :blocking, :movablekeys], 0, 0, 0, :stream},
"XREADGROUP" => {-7, [:write, :blocking, :movablekeys], 0, 0, 0, :stream},
"XINFO" => {-2, [:readonly], 0, 0, 0, :stream},
"XGROUP" => {-2, [:write, :denyoom], 0, 0, 0, :stream},
"SETBIT" => {4, [:write, :denyoom], 1, 1, 1, :bitmap},
"GETBIT" => {3, [:readonly, :fast], 1, 1, 1, :bitmap},
"BITCOUNT" => {-2, [:readonly], 1, 1, 1, :bitmap},
"BITPOS" => {-3, [:readonly], 1, 1, 1, :bitmap},
"BITFIELD" => {-2, [:write, :denyoom], 1, 1, 1, :bitmap},
"BITOP" => {-4, [:write, :denyoom], 2, -1, 1, :bitmap},
"PFADD" => {-2, [:write, :denyoom, :fast], 1, 1, 1, :hyperloglog},
"PFCOUNT" => {-2, [:readonly], 1, -1, 1, :hyperloglog},
"PFMERGE" => {-2, [:write, :denyoom], 1, -1, 1, :hyperloglog},
"SUBSCRIBE" => {-2, [:pubsub, :noscript, :loading, :stale, :fast], 0, 0, 0, :pubsub},
"UNSUBSCRIBE" => {-1, [:pubsub, :noscript, :loading, :stale, :fast], 0, 0, 0, :pubsub},
"PUBLISH" => {3, [:pubsub, :loading, :stale, :fast], 0, 0, 0, :pubsub},
"MULTI" => {1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :transactions},
"EXEC" => {1, [:noscript, :loading, :stale], 0, 0, 0, :transactions},
"DISCARD" => {1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :transactions},
"WATCH" => {-2, [:noscript, :loading, :stale, :fast], 1, -1, 1, :transactions},
"UNWATCH" => {1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :transactions}
}
@keyRules %{
"XREAD" => :streams,
"XREADGROUP" => :streams,
"SINTERCARD" => {:numkeys, 1},
"BLMPOP" => {:numkeys, 2},
"OBJECT" => {:subcommand, ~w(ENCODING REFCOUNT FREQ IDLETIME)},
"MEMORY" => {:subcommand, ~w(USAGE)},
"XINFO" => {:subcommand, ~w(STREAM GROUPS CONSUMERS)},
"XGROUP" => {:subcommand, ~w(CREATE SETID DESTROY CREATECONSUMER DELCONSUMER)}
}
def tableNames, do: Map.keys(@commands)
def names, do: Excache.Command.known_commands() |> Enum.uniq() |> Enum.sort()
def known?(name), do: Map.has_key?(@commands, name)
def count, do: length(names())
def infoTerm(name) do
case Map.fetch(@commands, name) do
{:ok, {arity, flags, firstKey, lastKey, step, _group}} ->
{:array,
[
{:bulk, String.downcase(name)},
{:integer, arity},
{:array, Enum.map(flags, fn flag -> {:simple, Atom.to_string(flag)} end)},
{:integer, firstKey},
{:integer, lastKey},
{:integer, step}
]}
:error ->
{:array, nil}
end
end
def docsTerm(name) do
{arity, _flags, _firstKey, _lastKey, _step, group} = Map.fetch!(@commands, name)
{:array,
[
{:bulk, "group"},
{:bulk, Atom.to_string(group)},
{:bulk, "arity"},
{:integer, arity}
]}
end
def arityOk?(name, argc) do
{arity, _flags, _firstKey, _lastKey, _step, _group} = Map.fetch!(@commands, name)
if arity < 0, do: argc >= -arity, else: argc == arity
end
def keysFor(name, args) do
case Map.get(@keyRules, name) do
nil -> rangeKeys(name, args)
rule -> ruleKeys(rule, args)
end
end
defp rangeKeys(name, args) do
{_arity, _flags, firstKey, lastKey, step, _group} = Map.fetch!(@commands, name)
if firstKey == 0 do
{:ok, []}
else
count = length(args)
from = firstKey - 1
to = if lastKey < 0, do: count + lastKey, else: lastKey - 1
if to < from do
{:ok, []}
else
{:ok, args |> Enum.slice(from..to) |> Enum.take_every(step)}
end
end
end
defp ruleKeys(:streams, args) do
case Enum.find_index(args, fn arg -> String.upcase(arg) == "STREAMS" end) do
nil ->
:error
index ->
tail = Enum.drop(args, index + 1)
half = div(length(tail), 2)
if half == 0 or rem(length(tail), 2) == 1, do: :error, else: {:ok, Enum.take(tail, half)}
end
end
defp ruleKeys({:numkeys, position}, args) do
with {numkeys, ""} when numkeys > 0 <- Integer.parse(Enum.at(args, position - 1, "")),
keys when length(keys) == numkeys <- args |> Enum.drop(position) |> Enum.take(numkeys) do
{:ok, keys}
else
_invalid -> :error
end
end
defp ruleKeys({:subcommand, takesKey}, [subcommand, key | _rest]) do
if String.upcase(subcommand) in takesKey, do: {:ok, [key]}, else: {:ok, []}
end
defp ruleKeys({:subcommand, _takesKey}, _args), do: {:ok, []}
end
+257
View File
@@ -0,0 +1,257 @@
defmodule Excache.Command.Introspection.Client do
@moduledoc false
alias Excache.Clients
alias Excache.Command.Reply
@help [
"CLIENT <subcommand> [<arg> [value] [opt] ...]. Subcommands are:",
"ID",
" Return the client id of the current connection.",
"INFO",
" Return information about the current connection.",
"GETNAME",
" Return the name of the current connection.",
"SETNAME <name>",
" Assign the name <name> to the current connection.",
"LIST [ID <id> [<id> ...]] [TYPE normal|master|replica|pubsub]",
" Return one line of information per connection.",
"KILL <ip:port>",
" Close the connection reachable at <ip:port>.",
"KILL <filter> <value> [<filter> <value> ...]",
" Close connections matching ID, ADDR, LADDR, TYPE, MAXAGE or SKIPME.",
"HELP",
" Print this help."
]
@killFilters ~w(ID ADDR LADDR TYPE MAXAGE SKIPME)
@clientTypes ~w(normal master replica pubsub)
def dispatch("ID", [], state),
do: withClient(state, fn client -> {:integer, client.id} end)
def dispatch("INFO", [], state),
do: withClient(state, fn client -> {:bulk, line(client, now())} end)
def dispatch("GETNAME", [], state),
do: withClient(state, fn client -> {:bulk, client.name} end)
def dispatch("SETNAME", [name], state) do
if validName?(name) do
withClient(state, fn client ->
:ok = Clients.setName(client.pid, name)
{:simple, "OK"}
end)
else
{[{:error, "ERR Client names cannot contain spaces, newlines or special characters."}],
state}
end
end
def dispatch("LIST", args, state) do
case listFilter(args) do
{:ok, matcher} ->
moment = now()
text =
Clients.snapshot().clients
|> Enum.filter(matcher)
|> Enum.map_join("", fn client -> line(client, moment) <> "\n" end)
{[{:bulk, text}], state}
:error ->
{[Reply.syntax()], state}
end
end
def dispatch("KILL", [address], state) do
killed = Clients.kill(fn client -> client.addr == address end, self())
if killed > 0 do
{[{:simple, "OK"}], state}
else
{[{:error, "ERR No such client address in the client list"}], state}
end
end
def dispatch("KILL", [_, _ | _] = args, state) do
case killFilter(args) do
{:ok, matcher} -> {[{:integer, Clients.kill(matcher, self())}], state}
:error -> {[Reply.syntax()], state}
end
end
def dispatch("HELP", [], state),
do: {[{:array, Enum.map(@help, fn text -> {:simple, text} end)}], state}
def dispatch(subcommand, _args, state) do
{[
{:error,
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
"Try CLIENT HELP."}
], state}
end
defp line(client, moment) do
Enum.join(
[
"id=#{client.id}",
"addr=#{client.addr}",
"laddr=#{client.laddr}",
"name=#{client.name}",
"age=#{elapsed(moment, client.connectedAt)}",
"idle=#{elapsed(moment, client.lastActiveAt)}",
"flags=N",
"db=0",
"sub=#{Clients.subscriptionCount(client.pid)}",
"psub=0",
"tot-net-in=#{client.netIn}",
"tot-net-out=#{client.netOut}"
],
" "
)
end
defp withClient(state, build) do
case Clients.find(self()) do
nil -> {[{:error, "ERR CLIENT is only available on a server connection"}], state}
client -> {[build.(client)], state}
end
end
defp listFilter([]), do: {:ok, fn _client -> true end}
defp listFilter([keyword | [_ | _] = rest]) do
case String.upcase(keyword) do
"ID" -> idFilter(rest)
"TYPE" -> typeFilter(rest)
_other -> :error
end
end
defp listFilter(_args), do: :error
defp idFilter(raw) do
ids = Enum.map(raw, &parseId/1)
if :error in ids do
:error
else
wanted = MapSet.new(ids)
{:ok, fn client -> MapSet.member?(wanted, client.id) end}
end
end
defp typeFilter([type]) do
case String.downcase(type) do
"normal" -> {:ok, fn client -> Clients.subscriptionCount(client.pid) == 0 end}
"pubsub" -> {:ok, fn client -> Clients.subscriptionCount(client.pid) > 0 end}
"master" -> {:ok, fn _client -> false end}
"replica" -> {:ok, fn _client -> false end}
_other -> :error
end
end
defp typeFilter(_raw), do: :error
defp killFilter(args) do
case parseFilters(args, %{skipme: true}) do
{:ok, filters} -> {:ok, buildMatcher(filters)}
:error -> :error
end
end
defp parseFilters([], filters), do: {:ok, filters}
defp parseFilters([keyword, value | rest], filters) do
upcased = String.upcase(keyword)
if upcased in @killFilters do
case addFilter(upcased, value, filters) do
{:ok, updated} -> parseFilters(rest, updated)
:error -> :error
end
else
:error
end
end
defp parseFilters(_args, _filters), do: :error
defp addFilter("ID", value, filters) do
case parseId(value) do
:error -> :error
id -> {:ok, Map.put(filters, :id, id)}
end
end
defp addFilter("ADDR", value, filters), do: {:ok, Map.put(filters, :addr, value)}
defp addFilter("LADDR", value, filters), do: {:ok, Map.put(filters, :laddr, value)}
defp addFilter("TYPE", value, filters) do
lowered = String.downcase(value)
if lowered in @clientTypes, do: {:ok, Map.put(filters, :type, lowered)}, else: :error
end
defp addFilter("MAXAGE", value, filters) do
case Integer.parse(value) do
{seconds, ""} when seconds >= 0 -> {:ok, Map.put(filters, :maxage, seconds)}
_invalid -> :error
end
end
defp addFilter("SKIPME", value, filters) do
case String.downcase(value) do
"yes" -> {:ok, Map.put(filters, :skipme, true)}
"no" -> {:ok, Map.put(filters, :skipme, false)}
_other -> :error
end
end
defp buildMatcher(filters) do
caller = self()
moment = now()
fn client ->
matchesId?(filters, client) and matchesAddr?(filters, client) and
matchesLaddr?(filters, client) and matchesType?(filters, client) and
matchesAge?(filters, client, moment) and matchesSkipme?(filters, client, caller)
end
end
defp matchesId?(%{id: id}, client), do: client.id == id
defp matchesId?(_filters, _client), do: true
defp matchesAddr?(%{addr: addr}, client), do: client.addr == addr
defp matchesAddr?(_filters, _client), do: true
defp matchesLaddr?(%{laddr: laddr}, client), do: client.laddr == laddr
defp matchesLaddr?(_filters, _client), do: true
defp matchesType?(%{type: "normal"}, client), do: Clients.subscriptionCount(client.pid) == 0
defp matchesType?(%{type: "pubsub"}, client), do: Clients.subscriptionCount(client.pid) > 0
defp matchesType?(%{type: _other}, _client), do: false
defp matchesType?(_filters, _client), do: true
defp matchesAge?(%{maxage: maxage}, client, moment),
do: elapsed(moment, client.connectedAt) >= maxage
defp matchesAge?(_filters, _client, _moment), do: true
defp matchesSkipme?(%{skipme: true}, client, caller), do: client.pid != caller
defp matchesSkipme?(_filters, _client, _caller), do: true
defp parseId(raw) do
case Integer.parse(raw) do
{id, ""} when id > 0 -> id
_invalid -> :error
end
end
defp validName?(name), do: not String.contains?(name, [" ", "\n", "\r"])
defp elapsed(moment, since), do: div(moment - since, 1000)
defp now, do: System.system_time(:millisecond)
end
+260
View File
@@ -0,0 +1,260 @@
defmodule Excache.Command.Introspection.Info do
@moduledoc false
alias Excache.Aof
alias Excache.Clients
alias Excache.Config
alias Excache.Eviction
alias Excache.Eviction.Runner
alias Excache.Memory
alias Excache.Persistence
alias Excache.Store
@redisCompat "7.2.0"
@sections %{
"server" => :server,
"clients" => :clients,
"memory" => :memory,
"persistence" => :persistence,
"stats" => :stats,
"replication" => :replication,
"keyspace" => :keyspace,
"cpu" => :cpu
}
@default [:server, :clients, :memory, :persistence, :stats, :replication, :keyspace]
@everything @default ++ [:cpu]
@titles %{
server: "Server",
clients: "Clients",
memory: "Memory",
persistence: "Persistence",
stats: "Stats",
replication: "Replication",
keyspace: "Keyspace",
cpu: "Cpu"
}
@clientSections [:server, :clients, :stats]
@keyspaceSections [:memory, :keyspace]
def render(args), do: args |> requested() |> emit()
defp requested([]), do: @default
defp requested(args) do
args
|> Enum.flat_map(fn arg ->
case String.downcase(arg) do
"all" -> @everything
"everything" -> @everything
"default" -> @default
name -> @sections |> Map.get(name) |> List.wrap()
end
end)
|> Enum.uniq()
end
defp emit([]), do: ""
defp emit(sections) do
context = context(sections)
sections
|> Enum.map(fn section -> block(Map.fetch!(@titles, section), lines(section, context)) end)
|> IO.iodata_to_binary()
end
defp block(title, fields) do
[
"# ",
title,
"\r\n",
Enum.map(fields, fn {key, value} -> [key, ":", text(value), "\r\n"] end),
"\r\n"
]
end
defp text(value) when is_binary(value), do: value
defp text(value) when is_integer(value), do: Integer.to_string(value)
defp text(value) when is_atom(value), do: Atom.to_string(value)
defp text(value) when is_list(value), do: List.to_string(value)
defp context(sections) do
%{}
|> withClients(Enum.any?(sections, &(&1 in @clientSections)))
|> withKeyspace(Enum.any?(sections, &(&1 in @keyspaceSections)))
end
defp withClients(context, false), do: context
defp withClients(context, true), do: Map.put(context, :clients, Clients.snapshot())
defp withKeyspace(context, false), do: context
defp withKeyspace(context, true) do
entries = Store.dump()
now = System.system_time(:millisecond)
remaining = for {_key, _kind, _value, at} <- entries, at != nil and at > now, do: at - now
Map.merge(context, %{
memory: Memory.stats(entries),
volatile: length(remaining),
avgTtl: averageTtl(remaining)
})
end
defp averageTtl([]), do: 0
defp averageTtl(remaining), do: div(Enum.sum(remaining), length(remaining))
defp lines(:server, context) do
snapshot = context.clients
uptime = div(System.system_time(:millisecond) - snapshot.startedAt, 1000)
[
{"excache_version", version()},
{"redis_version", @redisCompat},
{"redis_mode", "standalone"},
{"os", :erlang.system_info(:system_architecture)},
{"arch_bits", :erlang.system_info(:wordsize) * 8},
{"process_id", :os.getpid()},
{"run_id", snapshot.runId},
{"tcp_port", tcpPort()},
{"uptime_in_seconds", uptime},
{"uptime_in_days", div(uptime, 86_400)},
{"otp_release", :erlang.system_info(:otp_release)},
{"erts_version", :erlang.system_info(:version)},
{"elixir_version", System.version()}
]
end
defp lines(:clients, context) do
clients = context.clients.clients
[
{"connected_clients", length(clients)},
{"blocked_clients", Enum.count(clients, fn client -> Clients.blocked?(client.pid) end)},
{"pubsub_clients",
Enum.count(clients, fn client -> Clients.subscriptionCount(client.pid) > 0 end)}
]
end
defp lines(:memory, context) do
stats = context.memory
limit = Config.maxmemory()
[
{"used_memory", stats.usedMemory},
{"used_memory_human", humanBytes(stats.usedMemory)},
{"used_memory_dataset", stats.datasetBytes},
{"used_memory_overhead", stats.overheadBytes},
{"used_memory_beam", :erlang.memory(:total)},
{"used_memory_beam_processes", :erlang.memory(:processes)},
{"used_memory_beam_binary", :erlang.memory(:binary)},
{"used_memory_beam_ets", :erlang.memory(:ets)},
{"maxmemory", limit},
{"maxmemory_human", humanBytes(limit)},
{"maxmemory_policy", Eviction.policyName(Config.policy())},
{"maxmemory_samples", Config.maxmemorySamples()},
{"mem_allocator", "erts"}
]
end
defp lines(:persistence, _context) do
snapshotPath = Persistence.snapshot_path()
[
{"loading", 0},
{"rdb_last_save_time", fileMtime(snapshotPath)},
{"rdb_snapshot_size", fileSize(snapshotPath)},
{"aof_enabled", 1},
{"aof_rewrite_in_progress", 0},
{"aof_fsync_policy", Aof.fsync_policy()},
{"aof_current_size", fileSize(Aof.path())}
]
end
defp lines(:stats, context) do
snapshot = context.clients
runner = Runner.stats()
[
{"total_connections_received", snapshot.connectionsReceived},
{"total_net_input_bytes", snapshot.netIn},
{"total_net_output_bytes", snapshot.netOut},
{"evicted_keys", runner.evictedKeys},
{"oom_rejections", runner.oomRejections},
{"pubsub_channels", Clients.subscribedChannels()},
{"pubsub_patterns", 0}
]
end
defp lines(:replication, _context) do
[
{"role", "master"},
{"connected_slaves", 0},
{"master_failover_state", "no-failover"},
{"master_repl_offset", 0},
{"second_repl_offset", -1},
{"repl_backlog_active", 0},
{"repl_backlog_size", 0},
{"repl_backlog_first_byte_offset", 0},
{"repl_backlog_histlen", 0}
]
end
defp lines(:keyspace, context) do
case Store.dbsize() do
0 ->
[]
keys ->
[{"db0", "keys=#{keys},expires=#{context.volatile},avg_ttl=#{context.avgTtl}"}]
end
end
defp lines(:cpu, _context) do
{runtimeMillis, _sinceLast} = :erlang.statistics(:runtime)
{wallMillis, _sinceLast} = :erlang.statistics(:wall_clock)
[
{"used_cpu_total", seconds(runtimeMillis)},
{"wall_clock_seconds", seconds(wallMillis)},
{"schedulers", :erlang.system_info(:schedulers)},
{"schedulers_online", :erlang.system_info(:schedulers_online)},
{"run_queue", :erlang.statistics(:total_run_queue_lengths)}
]
end
defp version, do: :excache |> Application.spec(:vsn) |> List.to_string()
defp tcpPort do
{:ok, port} = Config.get("port")
port || 0
end
defp seconds(millis), do: :erlang.float_to_binary(millis / 1000, decimals: 2)
defp fileSize(path) do
case File.stat(path) do
{:ok, stat} -> stat.size
{:error, _reason} -> 0
end
end
defp fileMtime(path) do
case File.stat(path, time: :posix) do
{:ok, stat} -> stat.mtime
{:error, _reason} -> 0
end
end
defp humanBytes(bytes) when bytes < 1024, do: "#{bytes}B"
defp humanBytes(bytes) when bytes < 1_048_576, do: scaled(bytes, 1024, "K")
defp humanBytes(bytes) when bytes < 1_073_741_824, do: scaled(bytes, 1_048_576, "M")
defp humanBytes(bytes), do: scaled(bytes, 1_073_741_824, "G")
defp scaled(bytes, divisor, suffix),
do: :erlang.float_to_binary(bytes / divisor, decimals: 2) <> suffix
end
+504
View File
@@ -0,0 +1,504 @@
defmodule Excache.Command.Keyspace do
@moduledoc false
alias Excache.{SkipList, Store}
alias Excache.Command.Keyspace.Scan
alias Excache.Command.Reply
alias Excache.Eviction.Tracker
@scanFamily ~w(SCAN HSCAN SSCAN ZSCAN)
@sharedRefcount 2_147_483_647
@listpackEntries 128
@listpackValue 64
@intsetEntries 512
def dispatch(name, args, state) when name in @scanFamily,
do: Scan.dispatch(name, args, state)
def dispatch("OBJECT", [subcommand | rest], state),
do: object(String.upcase(subcommand), rest, state)
def dispatch("LPOS", [key, element | rest], state) do
case parseLposOptions(rest, %{rank: 1, count: nil, maxlen: 0}) do
{:ok, options} -> lpos(key, element, options, state)
{:reply, term} -> reply(term, state)
end
end
def dispatch("STRLEN", [key], state), do: reply(integerReply(Store.strlen(key)), state)
def dispatch("SETRANGE", [key, offsetRaw, value], state) do
case Integer.parse(offsetRaw) do
{offset, ""} when offset >= 0 -> reply(setrange(key, offset, value), state)
{_negative, ""} -> reply({:error, "ERR offset is out of range"}, state)
_ -> reply(Reply.not_integer(), state)
end
end
def dispatch("GETRANGE", [key, startRaw, stopRaw], state) do
Reply.with_two_ints(startRaw, stopRaw, state, fn start, stop ->
bulkReply(Store.getrange(key, start, stop))
end)
end
def dispatch("SETNX", [key, value], state),
do: reply({:integer, Store.setnx(key, value)}, state)
def dispatch("GETDEL", [key], state), do: reply(bulkReply(Store.getdel(key)), state)
def dispatch("INCRBYFLOAT", [key, deltaRaw], state) do
case Reply.parse_score(deltaRaw) do
{:ok, delta} -> reply(bulkReply(Store.incrbyfloat(key, delta)), state)
:error -> reply(Reply.not_float(), state)
end
end
def dispatch("HSETNX", [key, field, value], state),
do: reply(integerReply(Store.hsetnx(key, field, value)), state)
def dispatch("HMGET", [key | [_ | _] = fields], state) do
withType(key, "hash", state, fn ->
{:array, Reply.bulks_nullable(Store.hmget(key, fields))}
end)
end
def dispatch("SMISMEMBER", [key | [_ | _] = members], state) do
withType(key, "set", state, fn ->
{:array, Enum.map(Store.smismember(key, members), &{:integer, &1})}
end)
end
def dispatch("SINTERCARD", [numkeysRaw | [_ | _] = rest], state) do
case parseIntercard(numkeysRaw, rest) do
{:ok, keys, limit} -> reply(integerReply(Store.sintercard(keys, limit)), state)
{:reply, term} -> reply(term, state)
end
end
def dispatch("SPOP", [key], state), do: reply(singleReply(Store.spop(key, nil)), state)
def dispatch("SPOP", [key, countRaw], state) do
case Integer.parse(countRaw) do
{count, ""} when count >= 0 -> reply(arrayReply(Store.spop(key, count)), state)
{_negative, ""} -> reply({:error, "ERR value is out of range, must be positive"}, state)
_ -> reply(Reply.not_integer(), state)
end
end
def dispatch("SRANDMEMBER", [key], state),
do: reply(singleReply(Store.srandmember(key, nil)), state)
def dispatch("SRANDMEMBER", [key, countRaw], state) do
Reply.with_int(countRaw, state, fn count -> arrayReply(Store.srandmember(key, count)) end)
end
def dispatch("SMOVE", [source, dest, member], state) do
withType(dest, "set", state, fn -> integerReply(Store.smove(source, dest, member)) end)
end
def dispatch("LREM", [key, countRaw, value], state) do
Reply.with_int(countRaw, state, fn count -> integerReply(Store.lrem(key, count, value)) end)
end
def dispatch("LSET", [key, indexRaw, value], state) do
Reply.with_int(indexRaw, state, fn index -> statusReply(Store.lset(key, index, value)) end)
end
def dispatch("LTRIM", [key, startRaw, stopRaw], state) do
Reply.with_two_ints(startRaw, stopRaw, state, fn start, stop ->
statusReply(Store.ltrim(key, start, stop))
end)
end
def dispatch("LINSERT", [key, positionRaw, pivot, value], state) do
case String.upcase(positionRaw) do
"BEFORE" -> reply(integerReply(Store.linsert(key, :before, pivot, value)), state)
"AFTER" -> reply(integerReply(Store.linsert(key, :after, pivot, value)), state)
_ -> reply(Reply.syntax(), state)
end
end
def dispatch("RPOPLPUSH", [source, dest], state) do
withType(dest, "list", state, fn -> bulkReply(Store.rpoplpush(source, dest)) end)
end
def dispatch("ZCOUNT", [key, minRaw, maxRaw], state) do
case {Reply.parse_bound(minRaw), Reply.parse_bound(maxRaw)} do
{{:ok, minBound}, {:ok, maxBound}} ->
reply(integerReply(Store.zcount(key, minBound, maxBound)), state)
_ ->
reply({:error, "ERR min or max is not a float"}, state)
end
end
def dispatch("ZREVRANGE", [key, startRaw, stopRaw], state),
do: zrevrange(key, startRaw, stopRaw, false, state)
def dispatch("ZREVRANGE", [key, startRaw, stopRaw, flag], state) do
if String.upcase(flag) == "WITHSCORES" do
zrevrange(key, startRaw, stopRaw, true, state)
else
reply(Reply.syntax(), state)
end
end
def dispatch("ZREVRANK", [key, member], state),
do: reply(rankReply(Store.zrevrank(key, member)), state)
def dispatch("ZMSCORE", [key | [_ | _] = members], state) do
case Store.zmscore(key, members) do
{:ok, scores} -> reply({:array, Enum.map(scores, &scoreEntry/1)}, state)
{:error, reason} -> reply(errorTerm(reason), state)
end
end
def dispatch("RENAME", [source, dest], state),
do: reply(statusReply(Store.rename(source, dest)), state)
def dispatch("RENAMENX", [source, dest], state),
do: reply(integerReply(Store.renamenx(source, dest)), state)
def dispatch("COPY", [source, dest | rest], state) do
case parseCopyOptions(rest, false) do
{:ok, _replace} when source == dest ->
reply({:error, "ERR source and destination objects are the same"}, state)
{:ok, replace} ->
reply(integerReply(Store.copy(source, dest, replace)), state)
{:reply, term} ->
reply(term, state)
end
end
def dispatch("TOUCH", [_ | _] = keys, state), do: reply({:integer, Store.touch(keys)}, state)
def dispatch("UNLINK", [_ | _] = keys, state), do: reply({:integer, Store.del(keys)}, state)
def dispatch("EXPIREAT", [key, secondsRaw | rest], state) do
case Integer.parse(secondsRaw) do
{seconds, ""} ->
case parseExpireFlags(rest) do
{:ok, flags} -> reply({:integer, expireAt(key, seconds * 1000, flags)}, state)
{:reply, term} -> reply(term, state)
end
_ ->
reply(Reply.not_integer(), state)
end
end
def dispatch("EXPIRETIME", [key], state),
do: reply({:integer, Store.expiretime(key, :seconds)}, state)
def dispatch("PEXPIRETIME", [key], state),
do: reply({:integer, Store.expiretime(key, :milliseconds)}, state)
def dispatch(_name, _args, _state), do: :unhandled
defp setrange(key, _offset, ""), do: integerReply(Store.strlen(key))
defp setrange(key, offset, value), do: integerReply(Store.setrange(key, offset, value))
defp lpos(key, element, %{count: nil} = options, state) do
case Store.lpos(key, element, options.rank, 1, options.maxlen) do
{:ok, []} -> reply({:bulk, nil}, state)
{:ok, [index]} -> reply({:integer, index}, state)
{:error, reason} -> reply(errorTerm(reason), state)
end
end
defp lpos(key, element, options, state) do
case Store.lpos(key, element, options.rank, options.count, options.maxlen) do
{:ok, indexes} -> reply({:array, Enum.map(indexes, &{:integer, &1})}, state)
{:error, reason} -> reply(errorTerm(reason), state)
end
end
defp parseLposOptions([], options), do: {:ok, options}
defp parseLposOptions([token | rest], options) do
case {String.upcase(token), rest} do
{"RANK", [raw | tail]} ->
case Integer.parse(raw) do
{0, ""} -> {:reply, {:error, rankZeroMessage()}}
{rank, ""} -> parseLposOptions(tail, %{options | rank: rank})
_ -> {:reply, Reply.not_integer()}
end
{"COUNT", [raw | tail]} ->
case Integer.parse(raw) do
{count, ""} when count >= 0 -> parseLposOptions(tail, %{options | count: count})
{_negative, ""} -> {:reply, {:error, "ERR COUNT can't be negative"}}
_ -> {:reply, Reply.not_integer()}
end
{"MAXLEN", [raw | tail]} ->
case Integer.parse(raw) do
{maxlen, ""} when maxlen >= 0 -> parseLposOptions(tail, %{options | maxlen: maxlen})
{_negative, ""} -> {:reply, {:error, "ERR MAXLEN can't be negative"}}
_ -> {:reply, Reply.not_integer()}
end
_ ->
{:reply, Reply.syntax()}
end
end
defp rankZeroMessage do
"ERR RANK can't be zero. Use 1 to start searching from the first matching element in the " <>
"head of the list or -1 searching backward from the tail of the list"
end
defp object("ENCODING", [key], state) do
withEntry(key, state, fn entry -> {:bulk, encodingName(entry)} end)
end
defp object("REFCOUNT", [key], state) do
withEntry(key, state, fn entry -> {:integer, refcount(entry)} end)
end
defp object("IDLETIME", [key], state) do
withEntry(key, state, fn _entry -> {:integer, idleSeconds(key)} end)
end
defp object("HELP", [], state) do
lines = [
"OBJECT <subcommand> <key>. Subcommands are:",
"ENCODING <key>",
" Return the kind of internal representation used to store the value of <key>.",
"IDLETIME <key>",
" Return the seconds since <key> was last accessed, or 0 when the current",
" maxmemory-policy keeps no access metadata for it.",
"REFCOUNT <key>",
" Return the number of references of the value associated with <key>.",
"HELP",
" Print this help."
]
reply({:array, Enum.map(lines, &{:simple, &1})}, state)
end
defp object(subcommand, _rest, state) do
message =
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
"Try OBJECT HELP."
reply({:error, message}, state)
end
defp idleSeconds(key) do
case Tracker.idleMs(Tracker, key, System.system_time(:millisecond)) do
nil -> 0
idleMs -> div(idleMs, 1000)
end
end
defp withEntry(key, state, fun) do
case Store.encoding(key) do
nil -> reply({:error, "ERR no such key"}, state)
{:ok, entry} -> reply(fun.(entry), state)
end
end
defp encodingName({:string, value, _ea}) do
cond do
integerString?(value) -> "int"
byte_size(value) <= 44 -> "embstr"
true -> "raw"
end
end
defp encodingName({:list, value, _ea}) do
if compact?(value, length(value)), do: "listpack", else: "quicklist"
end
defp encodingName({:hash, value, _ea}) do
flat = Enum.flat_map(value, fn {field, item} -> [field, item] end)
if compact?(flat, map_size(value)), do: "listpack", else: "hashtable"
end
defp encodingName({:set, value, _ea}) do
members = MapSet.to_list(value)
size = MapSet.size(value)
cond do
size <= @intsetEntries and Enum.all?(members, &integerString?/1) -> "intset"
compact?(members, size) -> "listpack"
true -> "hashtable"
end
end
defp encodingName({:zset, value, _ea}) do
members = value |> SkipList.to_list() |> Enum.map(fn {_score, member} -> member end)
if compact?(members, SkipList.size(value)), do: "listpack", else: "skiplist"
end
defp encodingName({:stream, _value, _ea}), do: "stream"
defp compact?(values, size) do
size <= @listpackEntries and Enum.all?(values, &(byte_size(&1) <= @listpackValue))
end
defp refcount({:string, value, _ea}) do
if integerString?(value) and String.to_integer(value) in 0..9999 do
@sharedRefcount
else
1
end
end
defp refcount(_entry), do: 1
defp integerString?(value) do
case Integer.parse(value) do
{number, ""} -> Integer.to_string(number) == value and byte_size(value) <= 20
_ -> false
end
end
defp parseIntercard(numkeysRaw, rest) do
case Integer.parse(numkeysRaw) do
{numkeys, ""} when numkeys <= 0 ->
{:reply, {:error, "ERR numkeys should be greater than 0"}}
{numkeys, ""} when numkeys > length(rest) ->
{:reply, {:error, "ERR Number of keys can't be greater than number of args"}}
{numkeys, ""} ->
{keys, tail} = Enum.split(rest, numkeys)
case parseIntercardLimit(tail) do
{:ok, limit} -> {:ok, keys, limit}
{:reply, term} -> {:reply, term}
end
_ ->
{:reply, Reply.not_integer()}
end
end
defp parseIntercardLimit([]), do: {:ok, 0}
defp parseIntercardLimit([token, raw]) do
if String.upcase(token) == "LIMIT" do
case Integer.parse(raw) do
{limit, ""} when limit >= 0 -> {:ok, limit}
{_negative, ""} -> {:reply, {:error, "ERR LIMIT can't be negative"}}
_ -> {:reply, Reply.not_integer()}
end
else
{:reply, Reply.syntax()}
end
end
defp parseIntercardLimit(_tail), do: {:reply, Reply.syntax()}
defp parseCopyOptions([], replace), do: {:ok, replace}
defp parseCopyOptions([token | rest], replace) do
case {String.upcase(token), rest} do
{"REPLACE", tail} ->
parseCopyOptions(tail, true)
{"DB", [indexRaw | tail]} ->
case Integer.parse(indexRaw) do
{0, ""} -> parseCopyOptions(tail, replace)
{_other, ""} -> {:reply, {:error, "ERR DB index is out of range"}}
_ -> {:reply, Reply.not_integer()}
end
_ ->
{:reply, Reply.syntax()}
end
end
defp parseExpireFlags(tokens) do
tokens
|> Enum.reduce_while({:ok, %{nx: false, xx: false, gt: false, lt: false}}, fn token,
{:ok, flags} ->
case String.upcase(token) do
"NX" -> {:cont, {:ok, %{flags | nx: true}}}
"XX" -> {:cont, {:ok, %{flags | xx: true}}}
"GT" -> {:cont, {:ok, %{flags | gt: true}}}
"LT" -> {:cont, {:ok, %{flags | lt: true}}}
other -> {:halt, {:reply, {:error, "ERR Unsupported option #{other}"}}}
end
end)
|> validateExpireFlags()
end
defp validateExpireFlags({:reply, term}), do: {:reply, term}
defp validateExpireFlags({:ok, flags}) do
incompatible = (flags.nx and (flags.xx or flags.gt or flags.lt)) or (flags.gt and flags.lt)
if incompatible do
{:reply, {:error, "ERR NX and XX, GT or LT options at the same time are not compatible"}}
else
{:ok, flags}
end
end
defp expireAt(key, msAbs, flags) do
case Store.expiretime(key, :milliseconds) do
-2 -> 0
current -> if expireAllowed?(flags, current, msAbs), do: Store.expireat(key, msAbs), else: 0
end
end
defp expireAllowed?(flags, current, msAbs) do
cond do
flags.nx -> current == -1
flags.xx and current == -1 -> false
flags.gt -> current != -1 and msAbs > current
flags.lt -> current == -1 or msAbs < current
true -> true
end
end
defp zrevrange(key, startRaw, stopRaw, withscores, state) do
Reply.with_two_ints(startRaw, stopRaw, state, fn start, stop ->
Reply.zrange_reply(Store.zrevrange(key, start, stop), withscores)
end)
end
defp scoreEntry(nil), do: Reply.map_score(nil)
defp scoreEntry(score), do: Reply.map_score({:ok, score})
defp withType(key, expected, state, fun) do
case Store.type(key) do
"none" -> reply(fun.(), state)
^expected -> reply(fun.(), state)
_ -> reply(Reply.wrongtype(), state)
end
end
defp singleReply({:ok, [value | _rest]}), do: {:bulk, value}
defp singleReply({:ok, []}), do: {:bulk, nil}
defp singleReply({:error, reason}), do: errorTerm(reason)
defp arrayReply({:ok, values}), do: {:array, Reply.bulks(values)}
defp arrayReply({:error, reason}), do: errorTerm(reason)
defp integerReply({:ok, value}), do: {:integer, value}
defp integerReply({:error, reason}), do: errorTerm(reason)
defp bulkReply({:ok, value}), do: {:bulk, value}
defp bulkReply(nil), do: {:bulk, nil}
defp bulkReply({:error, reason}), do: errorTerm(reason)
defp rankReply({:ok, rank}), do: {:integer, rank}
defp rankReply(nil), do: {:bulk, nil}
defp rankReply({:error, reason}), do: errorTerm(reason)
defp statusReply({:ok, "OK"}), do: {:simple, "OK"}
defp statusReply({:error, reason}), do: errorTerm(reason)
defp errorTerm(:wrongtype), do: Reply.wrongtype()
defp errorTerm(:not_integer), do: Reply.not_integer()
defp errorTerm(:not_float), do: Reply.not_float()
defp errorTerm(:nokey), do: {:error, "ERR no such key"}
defp errorTerm(:range), do: {:error, "ERR index out of range"}
defp reply(term, state), do: {[term], state}
end
+183
View File
@@ -0,0 +1,183 @@
defmodule Excache.Command.Keyspace.Scan do
@moduledoc false
alias Excache.Command.Reply
alias Excache.Store
alias Excache.Store.Ops.Extra
@defaultCount 10
@buckets 64
def dispatch("SCAN", [cursorRaw | rest], state) do
with {:ok, cursor} <- parseCursor(cursorRaw),
{:ok, options} <- parseOptions(rest, [:match, :count, :type], defaults()) do
{next, keys} = walkShards(cursor, options)
reply(cursorReply(next, Reply.bulks(keys)), state)
else
{:reply, term} -> reply(term, state)
end
end
def dispatch("HSCAN", [key, cursorRaw | rest], state),
do: collection(:hash, key, cursorRaw, rest, state)
def dispatch("SSCAN", [key, cursorRaw | rest], state),
do: collection(:set, key, cursorRaw, rest, state)
def dispatch("ZSCAN", [key, cursorRaw | rest], state),
do: collection(:zset, key, cursorRaw, rest, state)
def dispatch(_name, _args, _state), do: :unhandled
defp collection(kind, key, cursorRaw, rest, state) do
with {:ok, cursor} <- parseCursor(cursorRaw),
{:ok, options} <- parseOptions(rest, allowedOptions(kind), defaults()),
{:ok, elements} <- fetchElements(kind, key) do
{next, selected} = walkBuckets(elements, cursor, options)
reply(cursorReply(next, renderElements(kind, selected, options)), state)
else
{:reply, term} -> reply(term, state)
end
end
defp allowedOptions(:hash), do: [:match, :count, :novalues]
defp allowedOptions(_kind), do: [:match, :count]
defp fetchElements(:hash, key) do
case Store.hgetall(key) do
{:ok, flat} ->
{:ok, flat |> Enum.chunk_every(2) |> Enum.map(fn [field, value] -> {field, value} end)}
{:error, :wrongtype} ->
{:reply, Reply.wrongtype()}
end
end
defp fetchElements(:set, key) do
case Store.smembers(key) do
{:ok, members} -> {:ok, Enum.map(members, &{&1, nil})}
{:error, :wrongtype} -> {:reply, Reply.wrongtype()}
end
end
defp fetchElements(:zset, key) do
case Store.zrange(key, 0, -1) do
{:ok, entries} -> {:ok, Enum.map(entries, fn {score, member} -> {member, score} end)}
{:error, :wrongtype} -> {:reply, Reply.wrongtype()}
end
end
defp renderElements(:hash, elements, %{novalues: true}),
do: Enum.map(elements, fn {field, _value} -> {:bulk, field} end)
defp renderElements(:hash, elements, _options),
do: Enum.flat_map(elements, fn {field, value} -> [{:bulk, field}, {:bulk, value}] end)
defp renderElements(:set, elements, _options),
do: Enum.map(elements, fn {member, _value} -> {:bulk, member} end)
defp renderElements(:zset, elements, _options) do
Enum.flat_map(elements, fn {member, score} ->
[{:bulk, member}, Reply.map_score({:ok, score})]
end)
end
defp walkShards(cursor, options) do
if cursor >= Store.shard_count() do
{0, []}
else
consumeShards(cursor, options, [])
end
end
defp consumeShards(cursor, options, acc) do
{next, keys} = Store.scan(cursor, options.match, options.type)
gathered = acc ++ keys
cond do
next == 0 -> {0, gathered}
length(gathered) >= options.count -> {next, gathered}
true -> consumeShards(next, options, gathered)
end
end
defp walkBuckets(_elements, cursor, _options) when cursor >= @buckets, do: {0, []}
defp walkBuckets(elements, cursor, options) do
grouped = Enum.group_by(elements, fn {member, _value} -> bucketOf(member) end)
consumeBuckets(grouped, cursor, options, [], 0)
end
defp consumeBuckets(grouped, bucket, options, acc, examined) do
chunk = Map.get(grouped, bucket, [])
selected = acc ++ matching(chunk, options.match)
seen = examined + length(chunk)
next = bucket + 1
cond do
next >= @buckets -> {0, selected}
seen >= options.count -> {next, selected}
true -> consumeBuckets(grouped, next, options, selected, seen)
end
end
defp bucketOf(member), do: rem(:erlang.phash2(member), @buckets)
defp matching(chunk, nil), do: chunk
defp matching(chunk, regex),
do: Enum.filter(chunk, fn {member, _value} -> Regex.match?(regex, member) end)
defp parseCursor(raw) do
case Integer.parse(raw) do
{cursor, ""} when cursor >= 0 -> {:ok, cursor}
_ -> {:reply, {:error, "ERR invalid cursor"}}
end
end
defp defaults, do: %{match: nil, count: @defaultCount, type: nil, novalues: false}
defp parseOptions([], _allowed, options), do: {:ok, options}
defp parseOptions([token | rest], allowed, options) do
case {String.upcase(token), rest} do
{"MATCH", [pattern | tail]} ->
withOption(:match, allowed, fn ->
parseOptions(tail, allowed, %{options | match: Extra.compilePattern(pattern)})
end)
{"COUNT", [raw | tail]} ->
withOption(:count, allowed, fn -> parseCount(raw, tail, allowed, options) end)
{"TYPE", [raw | tail]} ->
withOption(:type, allowed, fn ->
parseOptions(tail, allowed, %{options | type: String.downcase(raw)})
end)
{"NOVALUES", tail} ->
withOption(:novalues, allowed, fn ->
parseOptions(tail, allowed, %{options | novalues: true})
end)
_ ->
{:reply, Reply.syntax()}
end
end
defp withOption(name, allowed, fun) do
if name in allowed, do: fun.(), else: {:reply, Reply.syntax()}
end
defp parseCount(raw, tail, allowed, options) do
case Integer.parse(raw) do
{count, ""} when count > 0 -> parseOptions(tail, allowed, %{options | count: count})
{_tooSmall, ""} -> {:reply, Reply.syntax()}
_ -> {:reply, Reply.not_integer()}
end
end
defp cursorReply(cursor, items),
do: {:array, [{:bulk, Integer.to_string(cursor)}, {:array, items}]}
defp reply(term, state), do: {[term], state}
end
+280
View File
@@ -0,0 +1,280 @@
defmodule Excache.Command.Server do
@moduledoc false
alias Excache.Command.Reply
alias Excache.Config
alias Excache.Eviction
alias Excache.Eviction.Runner
alias Excache.Eviction.Tracker
alias Excache.Memory
alias Excache.Store
@doctorHeavyKey 25
@doctorNearLimit 80
def dispatch("MEMORY", [subcommand | rest], state),
do: memory(String.upcase(subcommand), rest, state)
def dispatch("MEMORY", [], state), do: {[wrongArity("memory")], state}
def dispatch("CONFIG", [subcommand | rest], state),
do: config(String.upcase(subcommand), rest, state)
def dispatch("CONFIG", [], state), do: {[wrongArity("config")], state}
defp memory("USAGE", [key], state), do: {[usage(key)], state}
defp memory("USAGE", [key, option, count], state) do
cond do
String.upcase(option) != "SAMPLES" -> {[Reply.syntax()], state}
not nonNegativeInteger?(count) -> {[Reply.not_integer()], state}
true -> {[usage(key)], state}
end
end
defp memory("STATS", [], state), do: {[{:array, statsArray()}], state}
defp memory("DOCTOR", [], state), do: {[{:bulk, doctor()}], state}
defp memory(subcommand, _args, state), do: {[unknownSubcommand("MEMORY", subcommand)], state}
defp config("GET", [_ | _] = patterns, state) do
values =
patterns
|> Enum.flat_map(&Config.getGlob/1)
|> Enum.uniq_by(fn {name, _value} -> name end)
|> Enum.sort()
|> Enum.flat_map(fn {name, value} -> [{:bulk, name}, {:bulk, value}] end)
{[{:array, values}], state}
end
defp config("GET", [], state), do: {[wrongArity("config|get")], state}
defp config("SET", [_, _ | _] = args, state) do
if rem(length(args), 2) == 0 do
{[applySettings(Reply.pairs(args))], state}
else
{[wrongArity("config|set")], state}
end
end
defp config("SET", _args, state), do: {[wrongArity("config|set")], state}
defp config("RESETSTAT", [], state) do
Runner.resetStats()
{[{:simple, "OK"}], state}
end
defp config(subcommand, _args, state), do: {[unknownSubcommand("CONFIG", subcommand)], state}
defp applySettings(pairs) do
case Config.setMany(pairs) do
:ok -> {:simple, "OK"}
{:error, name, reason} -> settingError(name, reason)
end
end
defp settingError(name, :unknown),
do: {:error, "ERR Unknown option or number of arguments for CONFIG SET - '#{name}'"}
defp settingError(name, :immutable),
do: {:error, "ERR CONFIG SET failed - can't set immutable config '#{name}'"}
defp settingError(_name, {:invalid, hint}), do: {:error, "ERR CONFIG SET failed - #{hint}"}
defp usage(key) do
case Memory.usage(key) do
nil -> {:bulk, nil}
bytes -> {:integer, bytes}
end
end
defp statsArray do
stats = Memory.stats(Store.dump())
runner = Runner.stats()
stats
|> statsPairs(runner)
|> Enum.flat_map(fn {name, term} -> [{:bulk, name}, term] end)
end
defp statsPairs(stats, runner) do
[
{"keys.count", {:integer, stats.keys}},
{"keys.volatile", {:integer, stats.volatileKeys}},
{"keys.bytes-per-key", {:integer, stats.bytesPerKey}},
{"total.allocated", {:integer, stats.usedMemory}},
{"dataset.bytes", {:integer, stats.datasetBytes}},
{"overhead.total", {:integer, stats.overheadBytes}},
{"largest.key", {:bulk, largestName(stats.largestKey)}},
{"largest.key.bytes", {:integer, largestBytes(stats.largestKey)}},
{"maxmemory", {:integer, Config.maxmemory()}},
{"maxmemory.policy", {:bulk, Eviction.policyName(Config.policy())}},
{"maxmemory.samples", {:integer, Config.maxmemorySamples()}},
{"evicted.keys", {:integer, runner.evictedKeys}},
{"oom.rejections", {:integer, runner.oomRejections}},
{"tracker.keys", {:integer, Tracker.size(Tracker)}}
] ++ typePairs(stats.byType)
end
defp typePairs(byType) do
Enum.flat_map(Memory.kinds(), fn kind ->
usage = Map.fetch!(byType, kind)
[
{"type.#{kind}.keys", {:integer, usage.keys}},
{"type.#{kind}.bytes", {:integer, usage.bytes}}
]
end)
end
defp largestName(nil), do: nil
defp largestName({key, _bytes}), do: key
defp largestBytes(nil), do: 0
defp largestBytes({_key, bytes}), do: bytes
defp doctor do
entries = Store.dump()
stats = Memory.stats(entries)
if stats.keys == 0 do
"Excache holds no keys, there is nothing to report."
else
report = %{
stats: stats,
limit: Config.maxmemory(),
policy: Config.policy(),
runner: Runner.stats(),
tracked: Tracker.size(Tracker)
}
case findings(report) do
[] -> headline(report) <> " Nothing looks wrong."
found -> Enum.join([headline(report) | found], "\n")
end
end
end
defp headline(%{stats: stats, policy: policy, limit: limit}) do
limitText = if limit == 0, do: "no limit", else: "#{limit} byte limit"
"#{stats.keys} keys, #{stats.usedMemory} bytes used, #{limitText}, " <>
"maxmemory-policy #{Eviction.policyName(policy)}."
end
defp findings(report) do
[
&noLimit/1,
&overLimit/1,
&nearLimit/1,
&noVolatileKeys/1,
&dominantKey/1,
&overheadHeavy/1,
&untrackedKeys/1,
&evictionActivity/1,
&refusedWrites/1
]
|> Enum.map(& &1.(report))
|> Enum.reject(&is_nil/1)
end
defp noLimit(%{limit: 0, stats: stats}) do
"maxmemory is not set, so nothing bounds the keyspace; it already holds " <>
"#{stats.usedMemory} bytes and will keep growing until the host runs out."
end
defp noLimit(_report), do: nil
defp overLimit(%{limit: limit, stats: stats, policy: :noeviction}) when limit > 0 do
if stats.usedMemory > limit do
"used memory #{stats.usedMemory} is over the #{limit} byte limit and " <>
"maxmemory-policy is noeviction, so every command that allocates is being refused."
end
end
defp overLimit(%{limit: limit, stats: stats}) when limit > 0 do
if stats.usedMemory > limit do
"used memory #{stats.usedMemory} is over the #{limit} byte limit, " <>
"the next write will free keys."
end
end
defp overLimit(_report), do: nil
defp nearLimit(%{limit: limit, stats: stats}) when limit > 0 do
share = percent(stats.usedMemory, limit)
if share >= @doctorNearLimit and stats.usedMemory <= limit do
"used memory is #{share} percent of the #{limit} byte limit, eviction is close."
end
end
defp nearLimit(_report), do: nil
defp noVolatileKeys(%{policy: policy, stats: stats, limit: limit}) when limit > 0 do
if Eviction.volatile?(policy) and stats.volatileKeys == 0 do
"maxmemory-policy #{Eviction.policyName(policy)} only evicts keys with a TTL and " <>
"none of the #{stats.keys} keys has one, so eviction cannot free anything."
end
end
defp noVolatileKeys(_report), do: nil
defp dominantKey(%{stats: %{largestKey: {key, bytes}} = stats}) do
share = percent(bytes, stats.usedMemory)
if share >= @doctorHeavyKey and stats.keys > 1 do
"key '#{key}' holds #{bytes} bytes, #{share} percent of everything stored."
end
end
defp dominantKey(_report), do: nil
defp overheadHeavy(%{stats: stats}) do
share = percent(stats.overheadBytes, stats.usedMemory)
if share > 50 do
"per key overhead is #{share} percent of the total at #{stats.bytesPerKey} bytes per key, " <>
"the keyspace is dominated by very small values."
end
end
defp overheadHeavy(_report), do: nil
defp untrackedKeys(%{policy: policy, stats: stats, tracked: tracked}) do
if policy in [:allkeysLru, :allkeysLfu, :volatileLru, :volatileLfu] and tracked < stats.keys do
"access metadata covers #{tracked} of #{stats.keys} keys, " <>
"the keys nobody has read yet are the first to go."
end
end
defp untrackedKeys(_report), do: nil
defp evictionActivity(%{runner: %{evictedKeys: evicted}}) when evicted > 0,
do: "#{evicted} keys have been evicted since the last CONFIG RESETSTAT."
defp evictionActivity(_report), do: nil
defp refusedWrites(%{runner: %{oomRejections: rejected}}) when rejected > 0,
do: "#{rejected} commands hit the memory limit since the last CONFIG RESETSTAT."
defp refusedWrites(_report), do: nil
defp percent(_part, 0), do: 0
defp percent(part, whole), do: round(part * 100 / whole)
defp nonNegativeInteger?(raw) do
match?({value, ""} when value >= 0, Integer.parse(raw))
end
defp wrongArity(name),
do: {:error, "ERR wrong number of arguments for '#{name}' command"}
defp unknownSubcommand(command, subcommand) do
{:error,
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
"Try #{command} HELP."}
end
end
+208
View File
@@ -0,0 +1,208 @@
defmodule Excache.Config do
@moduledoc false
alias Excache.Aof
alias Excache.Eviction
alias Excache.Eviction.Runner
alias Excache.Persistence
@defaults %{
maxmemory: 0,
maxmemoryPolicy: :noeviction,
maxmemorySamples: 5,
lfuLogFactor: 10,
lfuDecayTime: 1,
port: 6379,
snapshot_path: nil,
aof_path: nil
}
@parameters %{
"maxmemory" => %{env: :maxmemory, type: :bytes, mutable: true, runtime: true},
"maxmemory-policy" => %{env: :maxmemoryPolicy, type: :policy, mutable: true, runtime: true},
"maxmemory-samples" => %{
env: :maxmemorySamples,
type: {:bounded, 1, 64},
mutable: true,
runtime: true
},
"lfu-log-factor" => %{
env: :lfuLogFactor,
type: {:bounded, 0, 255},
mutable: true,
runtime: true
},
"lfu-decay-time" => %{
env: :lfuDecayTime,
type: {:bounded, 0, 1440},
mutable: true,
runtime: true
},
"port" => %{env: :port, type: {:bounded, 0, 65_535}, mutable: false, runtime: false},
"snapshot_path" => %{env: :snapshot_path, type: :path, mutable: true, runtime: false},
"aof_path" => %{env: :aof_path, type: :path, mutable: false, runtime: false}
}
@trackedPolicies [:allkeysLru, :allkeysLfu, :volatileLru, :volatileLfu]
@units %{
"" => 1,
"b" => 1,
"k" => 1000,
"kb" => 1024,
"m" => 1_000_000,
"mb" => 1_048_576,
"g" => 1_000_000_000,
"gb" => 1_073_741_824
}
def names, do: @parameters |> Map.keys() |> Enum.sort()
def get(name) do
case Map.fetch(@parameters, String.downcase(name)) do
{:ok, spec} -> {:ok, readSpec(spec)}
:error -> :error
end
end
def set(name, raw) do
case prepare(name, raw) do
{:ok, change} -> commit(change)
{:error, reason} -> {:error, reason}
end
end
def setMany(pairs) do
pairs
|> Enum.reduce_while({:ok, []}, fn {name, raw}, {:ok, acc} ->
case prepare(name, raw) do
{:ok, change} -> {:cont, {:ok, [change | acc]}}
{:error, reason} -> {:halt, {:error, name, reason}}
end
end)
|> case do
{:ok, changes} ->
changes |> Enum.reverse() |> Enum.each(&commit/1)
:ok
failure ->
failure
end
end
def getGlob(pattern) do
chars = pattern |> String.downcase() |> String.to_charlist()
for name <- names(), matches?(chars, String.to_charlist(name)) do
{:ok, value} = get(name)
{name, format(value)}
end
end
def reset do
for {_name, spec} <- @parameters, spec.runtime do
Application.delete_env(:excache, spec.env)
end
Runner.invalidate()
Runner.reloadTracker()
:ok
end
defp format(nil), do: ""
defp format(value) when is_integer(value), do: Integer.to_string(value)
defp format(value) when is_binary(value), do: value
defp format(policy) when is_atom(policy), do: Eviction.policyName(policy)
def maxmemory, do: readEnv(:maxmemory)
def policy, do: readEnv(:maxmemoryPolicy)
def maxmemorySamples, do: readEnv(:maxmemorySamples)
def lfuLogFactor, do: readEnv(:lfuLogFactor)
def lfuDecayTime, do: readEnv(:lfuDecayTime)
def tracksAccess?, do: policy() in @trackedPolicies
defp hint(:bytes), do: "argument must be a memory value like 512, 64kb, 100mb or 1gb"
defp hint(:policy),
do:
"argument must be one of the following: " <>
Enum.map_join(Eviction.policies(), ", ", &Eviction.policyName/1)
defp hint({:bounded, low, high}), do: "argument must be an integer between #{low} and #{high}"
defp hint(:path), do: "argument must be a non empty path"
defp prepare(name, raw) do
case Map.fetch(@parameters, String.downcase(name)) do
:error ->
{:error, :unknown}
{:ok, %{mutable: false}} ->
{:error, :immutable}
{:ok, spec} ->
case parse(spec.type, raw) do
{:ok, value} -> {:ok, {spec, value}}
:error -> {:error, {:invalid, hint(spec.type)}}
end
end
end
defp commit({spec, value}) do
previous = readSpec(spec)
Application.put_env(:excache, spec.env, value)
if value != previous, do: applyChange(spec)
:ok
end
defp applyChange(%{env: env}) when env in [:lfuLogFactor, :lfuDecayTime],
do: Runner.reloadTracker()
defp applyChange(%{env: env}) when env in [:maxmemory, :maxmemoryPolicy, :maxmemorySamples],
do: Runner.invalidate()
defp applyChange(_spec), do: :ok
defp parse(:bytes, raw), do: parseBytes(raw)
defp parse(:policy, raw), do: Eviction.parsePolicy(String.downcase(raw))
defp parse({:bounded, low, high}, raw) do
case Integer.parse(raw) do
{value, ""} when value >= low and value <= high -> {:ok, value}
_ -> :error
end
end
defp parse(:path, ""), do: :error
defp parse(:path, raw), do: {:ok, raw}
defp parseBytes(raw) do
{digits, unit} =
raw
|> String.downcase()
|> String.to_charlist()
|> Enum.split_while(&(&1 in ?0..?9))
with {:ok, factor} <- Map.fetch(@units, List.to_string(unit)),
{amount, ""} <- Integer.parse(List.to_string(digits)) do
{:ok, amount * factor}
else
_ -> :error
end
end
defp readSpec(%{env: :snapshot_path}), do: Persistence.snapshot_path()
defp readSpec(%{env: :aof_path}), do: Aof.path()
defp readSpec(%{env: env}), do: readEnv(env)
defp readEnv(env), do: Application.get_env(:excache, env, @defaults[env])
defp matches?([], name), do: name == []
defp matches?([?* | pattern], name),
do: matches?(pattern, name) or (name != [] and matches?([?* | pattern], tl(name)))
defp matches?([?? | pattern], [_char | name]), do: matches?(pattern, name)
defp matches?([char | pattern], [char | name]), do: matches?(pattern, name)
defp matches?(_pattern, _name), do: false
end
+259
View File
@@ -0,0 +1,259 @@
defmodule Excache.Eviction do
@moduledoc false
# Redis style maxmemory policies over the Excache keyspace.
#
# Nothing here sorts the whole keyspace. Every pass draws maxmemory-samples
# random candidates, scores them under the active policy and merges them into
# a bounded pool that survives between passes, so a strong candidate seen in
# an earlier pass still wins against a weak fresh sample. The victim is the
# pool head; the loop repeats until the estimated usage drops under the limit
# or the candidate set runs dry.
#
# Both the clock and the sampling function are injectable, so a run is fully
# reproducible. The default keyspace adapter reads the whole store once per
# run and then accounts freed bytes incrementally, the same estimate driven
# approach Redis uses inside performEvictions.
alias Excache.Eviction.Tracker
alias Excache.Memory
alias Excache.Store
@policies [
:noeviction,
:allkeysLru,
:allkeysLfu,
:allkeysRandom,
:volatileLru,
:volatileLfu,
:volatileTtl,
:volatileRandom
]
@policyNames %{
noeviction: "noeviction",
allkeysLru: "allkeys-lru",
allkeysLfu: "allkeys-lfu",
allkeysRandom: "allkeys-random",
volatileLru: "volatile-lru",
volatileLfu: "volatile-lfu",
volatileTtl: "volatile-ttl",
volatileRandom: "volatile-random"
}
@volatilePolicies [:volatileLru, :volatileLfu, :volatileTtl, :volatileRandom]
@allkeysPolicies [:allkeysLru, :allkeysLfu, :allkeysRandom]
@randomPolicies [:allkeysRandom, :volatileRandom]
@lruPolicies [:allkeysLru, :volatileLru]
@lfuPolicies [:allkeysLfu, :volatileLfu]
@defaultSampleSize 5
@defaultPoolSize 16
@defaultSeed {5, 9, 27}
@lfuMaxCounter 255
def policies, do: @policies
def policyName(policy), do: Map.fetch!(@policyNames, policy)
def parsePolicy(name) when is_binary(name) do
case Enum.find(@policies, fn policy -> @policyNames[policy] == name end) do
nil -> :error
policy -> {:ok, policy}
end
end
def volatile?(policy), do: policy in @volatilePolicies
def allkeys?(policy), do: policy in @allkeysPolicies
def random?(policy), do: policy in @randomPolicies
def defaultSampleSize, do: @defaultSampleSize
def defaultPoolSize, do: @defaultPoolSize
def storeKeyspace do
%{entries: &Store.dump/0, delete: fn key -> Store.del([key]) end}
end
def candidates(entries, policy) do
entries
|> Enum.map(fn {key, kind, value, expiresAt} ->
%{
key: key,
kind: kind,
expiresAt: expiresAt,
bytes: Memory.estimateEntry(key, {kind, value, expiresAt})
}
end)
|> Enum.filter(&eligible?(&1, policy))
end
def selectVictim(entries, opts) do
config = configure(opts)
pool = candidates(entries, config.policy)
case pool do
[] ->
{:error, emptyReason(config.policy)}
_ ->
{victim, _pool, _config} = pickVictim(pool, [], config)
{:ok, victim.key}
end
end
def run(opts) do
config = configure(opts)
keyspace = Keyword.get_lazy(opts, :keyspace, &storeKeyspace/0)
entries = keyspace.entries.()
pool = candidates(entries, config.policy)
acc = %{used: Memory.usedMemory(entries), evicted: [], freed: 0, passes: 0}
cond do
config.maxmemory <= 0 or acc.used <= config.maxmemory ->
{:ok, report(config, acc, :withinLimit)}
config.policy == :noeviction ->
{:error, report(config, acc, :noeviction)}
pool == [] ->
{:error, report(config, acc, emptyReason(config.policy))}
true ->
evictLoop(pool, [], config, acc, keyspace)
end
end
defp evictLoop(remaining, pool, config, acc, keyspace) do
cond do
acc.used <= config.maxmemory ->
{:ok, report(config, acc, :evicted)}
remaining == [] ->
{:error, report(config, acc, :exhausted)}
true ->
{victim, carried, config} = pickVictim(remaining, pool, config)
keyspace.delete.(victim.key)
if config.tracker, do: Tracker.forget(config.tracker, victim.key)
acc = %{
acc
| used: acc.used - victim.bytes,
freed: acc.freed + victim.bytes,
evicted: [victim.key | acc.evicted],
passes: acc.passes + 1
}
nextRemaining = Enum.reject(remaining, &(&1.key == victim.key))
nextPool = Enum.reject(carried, fn {_score, held} -> held.key == victim.key end)
evictLoop(nextRemaining, nextPool, config, acc, keyspace)
end
end
defp pickVictim(remaining, pool, config) do
{sampled, randState} = config.sampler.(remaining, config.sampleSize, config.randState)
config = %{config | randState: randState}
if random?(config.policy) do
{hd(sampled), pool, config}
else
metadata = config.metaFun.(Enum.map(sampled, & &1.key), config.now)
scored =
Enum.map(sampled, fn candidate ->
{score(config.policy, candidate, Map.get(metadata, candidate.key), config.now),
candidate}
end)
[{_score, victim} | rest] = mergePool(pool, scored, config.poolSize)
{victim, rest, config}
end
end
defp mergePool(pool, scored, poolSize) do
(scored ++ pool)
|> Enum.uniq_by(fn {_score, candidate} -> candidate.key end)
|> Enum.sort_by(fn {candidateScore, _candidate} -> candidateScore end, :desc)
|> Enum.take(poolSize)
end
defp score(policy, _candidate, meta, now) when policy in @lruPolicies, do: lruScore(meta, now)
defp score(policy, _candidate, meta, _now) when policy in @lfuPolicies,
do: @lfuMaxCounter - counterOf(meta)
defp score(:volatileTtl, candidate, _meta, _now), do: -candidate.expiresAt
defp lruScore(nil, _now), do: {1, 0}
defp lruScore(meta, now), do: {0, max(now - meta.lastAccess, 0)}
defp counterOf(nil), do: 0
defp counterOf(meta), do: meta.counter
defp eligible?(_candidate, :noeviction), do: false
defp eligible?(candidate, policy) do
if volatile?(policy), do: candidate.expiresAt != nil, else: true
end
defp emptyReason(policy) do
if volatile?(policy), do: :noVolatileKeys, else: :noCandidates
end
defp configure(opts) do
%{
policy: Keyword.fetch!(opts, :policy),
maxmemory: Keyword.get(opts, :maxmemory, 0),
now: Keyword.get_lazy(opts, :now, fn -> System.system_time(:millisecond) end),
sampleSize: Keyword.get(opts, :sampleSize, @defaultSampleSize),
poolSize: Keyword.get(opts, :poolSize, @defaultPoolSize),
sampler: Keyword.get(opts, :sampler, &sampleRandom/3),
randState: :rand.seed_s(:exsss, Keyword.get(opts, :seed, @defaultSeed)),
tracker: Keyword.get(opts, :tracker),
metaFun: metaFun(opts)
}
end
defp metaFun(opts) do
cond do
Keyword.has_key?(opts, :metadata) ->
metadata = Keyword.fetch!(opts, :metadata)
fn keys, _now -> Map.take(metadata, keys) end
Keyword.has_key?(opts, :tracker) ->
tracker = Keyword.fetch!(opts, :tracker)
fn keys, now -> Tracker.metadata(tracker, keys, now) end
true ->
fn _keys, _now -> %{} end
end
end
defp sampleRandom(pool, count, randState) when length(pool) <= count, do: {pool, randState}
defp sampleRandom(pool, count, randState) do
{taken, _rest, _size, nextState} =
Enum.reduce(1..count, {[], pool, length(pool), randState}, fn _index,
{acc, left, size, state} ->
{index, drawn} = :rand.uniform_s(size, state)
{picked, rest} = List.pop_at(left, index - 1)
{[picked | acc], rest, size - 1, drawn}
end)
{Enum.reverse(taken), nextState}
end
defp report(config, acc, reason) do
%{
reason: reason,
policy: config.policy,
maxmemory: config.maxmemory,
usedMemory: acc.used,
freedBytes: acc.freed,
evicted: Enum.reverse(acc.evicted),
evictedCount: length(acc.evicted),
passes: acc.passes
}
end
end
+208
View File
@@ -0,0 +1,208 @@
defmodule Excache.Eviction.Runner do
@moduledoc false
use GenServer
alias Excache.Aof
alias Excache.Config
alias Excache.Eviction
alias Excache.Eviction.Tracker
alias Excache.Memory
alias Excache.Notify
alias Excache.Store
@meter {__MODULE__, :meter}
@used 1
@dirty 2
@checkedAt 3
@verdict 4
@slots 4
@defaultInterval 200
@defaultRecheck 20
def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
def admit do
limit = Config.maxmemory()
if limit <= 0, do: :ok, else: decide(limit)
end
def account(args) do
if Config.maxmemory() > 0, do: :counters.add(meter(), @dirty, argBytes(args))
:ok
end
def enforce, do: GenServer.call(__MODULE__, :enforce, 30_000)
def invalidate do
meter = meter()
:counters.put(meter, @dirty, 0)
:counters.put(meter, @checkedAt, 0)
:ok
end
def reloadTracker, do: GenServer.call(__MODULE__, :reloadTracker)
def stats, do: GenServer.call(__MODULE__, :stats)
def resetStats, do: GenServer.call(__MODULE__, :resetStats)
@impl true
def init(_opts) do
:persistent_term.put(@meter, :counters.new(@slots, [:write_concurrency]))
schedule()
{:ok, %{evictedKeys: 0, oomRejections: 0}}
end
@impl true
def handle_call(:enforce, _from, state) do
{verdict, state} = sweep(state)
{:reply, verdict, state}
end
@impl true
def handle_call(:reloadTracker, _from, state) do
wanted = %{logFactor: Config.lfuLogFactor(), decayTime: Config.lfuDecayTime()}
if Tracker.settings(Tracker) != wanted, do: restartTracker()
{:reply, :ok, state}
end
@impl true
def handle_call(:stats, _from, state) do
{:reply, %{evictedKeys: state.evictedKeys, oomRejections: state.oomRejections}, state}
end
@impl true
def handle_call(:resetStats, _from, state),
do: {:reply, :ok, %{state | evictedKeys: 0, oomRejections: 0}}
@impl true
def handle_info(:tick, state) do
state = maintain(state)
schedule()
{:noreply, state}
end
defp maintain(state) do
cond do
Config.maxmemory() > 0 ->
elem(sweep(state), 1)
Config.tracksAccess?() ->
pruneMetadata(Store.dbsize(), fn -> Store.keys("*") end)
state
true ->
state
end
end
defp decide(limit) do
meter = meter()
checkedAt = :counters.get(meter, @checkedAt)
estimated = :counters.get(meter, @used) + :counters.get(meter, @dirty)
cond do
checkedAt == 0 -> enforce()
estimated <= limit -> :ok
recent?(checkedAt) -> cached(meter)
true -> enforce()
end
end
defp recent?(checkedAt) do
window = Application.get_env(:excache, :eviction_recheck, @defaultRecheck)
window > 0 and System.system_time(:millisecond) - checkedAt < window
end
defp cached(meter), do: if(:counters.get(meter, @verdict) == 0, do: :ok, else: :oom)
defp sweep(state) do
entries = Store.dump()
pruneMetadata(length(entries), fn -> Enum.map(entries, &elem(&1, 0)) end)
used = Memory.usedMemory(entries)
limit = Config.maxmemory()
policy = Config.policy()
cond do
limit <= 0 or used <= limit ->
record(used, :ok)
{:ok, state}
policy == :noeviction ->
record(used, :oom)
{:oom, %{state | oomRejections: state.oomRejections + 1}}
true ->
evict(entries, policy, limit, state)
end
end
defp evict(entries, policy, limit, state) do
{status, report} =
Eviction.run(
policy: policy,
maxmemory: limit,
sampleSize: Config.maxmemorySamples(),
tracker: Tracker,
keyspace: keyspace(entries)
)
verdict = if status == :ok, do: :ok, else: :oom
record(report.usedMemory, verdict)
state = %{
state
| evictedKeys: state.evictedKeys + report.evictedCount,
oomRejections: state.oomRejections + if(verdict == :oom, do: 1, else: 0)
}
{verdict, state}
end
defp keyspace(entries) do
%{
entries: fn -> entries end,
delete: fn key ->
deleted = Store.del([key])
Aof.append("DEL", [key])
Notify.key_event(key, "evicted")
deleted
end
}
end
defp pruneMetadata(keyCount, keysFun) do
if Config.tracksAccess?() and Tracker.size(Tracker) > keyCount do
Tracker.retain(Tracker, keysFun.())
end
:ok
end
defp restartTracker do
Supervisor.terminate_child(Excache.Supervisor, Tracker)
Supervisor.delete_child(Excache.Supervisor, Tracker)
Supervisor.start_child(Excache.Supervisor, Excache.Application.trackerSpec())
end
defp record(used, verdict) do
meter = meter()
:counters.put(meter, @used, used)
:counters.put(meter, @dirty, 0)
:counters.put(meter, @checkedAt, System.system_time(:millisecond))
:counters.put(meter, @verdict, if(verdict == :ok, do: 0, else: 1))
end
defp argBytes(args),
do: Enum.reduce(args, Memory.entryOverhead(), fn arg, total -> total + byte_size(arg) end)
defp meter, do: :persistent_term.get(@meter)
defp schedule do
interval = Application.get_env(:excache, :eviction_interval, @defaultInterval)
Process.send_after(self(), :tick, interval)
end
end
+187
View File
@@ -0,0 +1,187 @@
defmodule Excache.Eviction.Tracker do
@moduledoc false
# Per key access metadata for the LRU and LFU policies.
#
# The LFU counter follows Redis evict.c: it starts at LFU_INIT_VAL, grows
# logarithmically so a key read a million times cannot bury one read a
# thousand times, and decays by one unit for every lfu-decay-time minutes a
# key goes untouched. Both knobs and the random source are injectable so the
# behaviour is reproducible in tests; every timestamp is epoch milliseconds
# supplied by the caller, the tracker never reads the clock itself.
use GenServer
@lfuInitCounter 5
@lfuMaxCounter 255
@defaultLogFactor 10
@defaultDecayTime 1
@defaultSeed {1721, 4093, 8677}
def lfuInitCounter, do: @lfuInitCounter
def lfuMaxCounter, do: @lfuMaxCounter
def start_link(opts \\ []) do
{serverOpts, initOpts} = Keyword.split(opts, [:name])
GenServer.start_link(__MODULE__, initOpts, serverOpts)
end
def touch(server, key, now), do: GenServer.call(server, {:touch, key, now})
def touchAsync(server, key, now), do: GenServer.cast(server, {:touch, key, now})
def touchMany(server, keys, now), do: GenServer.call(server, {:touchMany, keys, now})
def forget(server, key), do: GenServer.call(server, {:forget, key})
def retain(server, keys), do: GenServer.call(server, {:retain, keys})
def clear(server), do: GenServer.call(server, :clear)
def size(server), do: GenServer.call(server, :size)
def lookup(server, key, now), do: GenServer.call(server, {:lookup, key, now})
def metadata(server, keys, now), do: GenServer.call(server, {:metadata, keys, now})
def snapshot(server, now), do: GenServer.call(server, {:snapshot, now})
def counter(server, key, now) do
case lookup(server, key, now) do
nil -> 0
meta -> meta.counter
end
end
def idleMs(server, key, now) do
case lookup(server, key, now) do
nil -> nil
meta -> meta.idleMs
end
end
def settings(server), do: GenServer.call(server, :settings)
@impl true
def init(opts) do
{:ok,
%{
access: %{},
logFactor: Keyword.get(opts, :logFactor, @defaultLogFactor),
decayTime: Keyword.get(opts, :decayTime, @defaultDecayTime),
randState: :rand.seed_s(:exsss, Keyword.get(opts, :seed, @defaultSeed))
}}
end
@impl true
def handle_call({:touch, key, now}, _from, state) do
{counter, state} = register(state, key, now)
{:reply, counter, state}
end
@impl true
def handle_call({:touchMany, keys, now}, _from, state) do
{counters, state} =
Enum.reduce(keys, {%{}, state}, fn key, {acc, current} ->
{counter, next} = register(current, key, now)
{Map.put(acc, key, counter), next}
end)
{:reply, counters, state}
end
@impl true
def handle_call({:forget, key}, _from, state),
do: {:reply, :ok, %{state | access: Map.delete(state.access, key)}}
@impl true
def handle_call({:retain, keys}, _from, state),
do: {:reply, :ok, %{state | access: Map.take(state.access, keys)}}
@impl true
def handle_call(:clear, _from, state), do: {:reply, :ok, %{state | access: %{}}}
@impl true
def handle_call(:size, _from, state), do: {:reply, map_size(state.access), state}
@impl true
def handle_call({:lookup, key, now}, _from, state) do
reply =
case Map.fetch(state.access, key) do
:error -> nil
{:ok, meta} -> present(meta, now, state.decayTime)
end
{:reply, reply, state}
end
@impl true
def handle_call({:metadata, keys, now}, _from, state) do
reply =
state.access
|> Map.take(keys)
|> Map.new(fn {key, meta} -> {key, present(meta, now, state.decayTime)} end)
{:reply, reply, state}
end
@impl true
def handle_call({:snapshot, now}, _from, state) do
reply =
Map.new(state.access, fn {key, meta} -> {key, present(meta, now, state.decayTime)} end)
{:reply, reply, state}
end
@impl true
def handle_call(:settings, _from, state) do
{:reply, %{logFactor: state.logFactor, decayTime: state.decayTime}, state}
end
@impl true
def handle_cast({:touch, key, now}, state) do
{_counter, state} = register(state, key, now)
{:noreply, state}
end
defp register(state, key, now) do
{counter, randState} =
case Map.fetch(state.access, key) do
:error ->
{@lfuInitCounter, state.randState}
{:ok, meta} ->
meta.counter
|> decay(meta.lastAccess, now, state.decayTime)
|> logIncrement(state.logFactor, state.randState)
end
access = Map.put(state.access, key, %{lastAccess: now, counter: counter})
{counter, %{state | access: access, randState: randState}}
end
defp logIncrement(@lfuMaxCounter, _logFactor, randState), do: {@lfuMaxCounter, randState}
defp logIncrement(counter, logFactor, randState) do
{roll, nextState} = :rand.uniform_s(randState)
baseval = max(counter - @lfuInitCounter, 0)
probability = 1.0 / (baseval * logFactor + 1)
if roll < probability, do: {counter + 1, nextState}, else: {counter, nextState}
end
defp decay(counter, _lastAccess, _now, decayTime) when decayTime <= 0, do: counter
defp decay(counter, lastAccess, now, decayTime) do
periods = div(div(max(now - lastAccess, 0), 60_000), decayTime)
max(counter - periods, 0)
end
defp present(meta, now, decayTime) do
%{
lastAccess: meta.lastAccess,
idleMs: max(now - meta.lastAccess, 0),
counter: decay(meta.counter, meta.lastAccess, now, decayTime)
}
end
end
+188
View File
@@ -0,0 +1,188 @@
defmodule Excache.Memory do
@moduledoc false
# Byte accounting for stored entries, the input every eviction policy needs.
#
# Structural cost comes from :erts_debug.size/1, which walks the term and
# counts machine words (shared subterms once). Two things it cannot see are
# added explicitly:
#
# * reference counted binaries. ERTS keeps binaries up to 64 bytes on the
# heap, so :erts_debug.size/1 covers those; anything larger lives
# off-heap and only its ProcBin header is counted, so the payload is
# added by walking the term.
#
# * the per-row cost of living in an ETS set. Measured against
# :ets.info(table, :memory) deltas on 64-bit OTP it is 41.24 bytes per
# row above the flat term size: five words of hash bucket bookkeeping
# plus an amortised bucket slot. Together with the five words of the
# {key, kind, value, expiresAt} tuple itself (header plus four slots)
# that is the ten word constant below; memory_test.exs asserts the
# resulting estimate against a real ETS insert.
alias Excache.SkipList
alias Excache.Store
alias Excache.Stream, as: CacheStream
@kinds [:string, :list, :hash, :set, :zset, :stream]
@entryOverheadWords 10
@heapBinaryLimit 64
def kinds, do: @kinds
def wordSize, do: :erlang.system_info(:wordsize)
def entryOverhead, do: @entryOverheadWords * wordSize()
def keyBytes(key) when is_binary(key), do: termBytes(key)
def estimateEntry(key, {kind, value, _expiresAt}) do
keyBytes(key) + estimateValue(kind, value) + entryOverhead()
end
def entryBytes({key, kind, value, expiresAt}),
do: estimateEntry(key, {kind, value, expiresAt})
def estimateValue(:string, value) when is_binary(value), do: termBytes(value)
def estimateValue(:list, value) when is_list(value), do: termBytes(value)
def estimateValue(:set, %MapSet{} = value), do: termBytes(value)
def estimateValue(:zset, %SkipList{} = value), do: termBytes(value)
def estimateValue(:stream, %CacheStream{} = value), do: termBytes(value)
def estimateValue(:hash, value) when is_map(value), do: termBytes(value)
def elementCount(:string, value) when is_binary(value), do: 1
def elementCount(:list, value), do: length(value)
def elementCount(:set, value), do: MapSet.size(value)
def elementCount(:zset, value), do: SkipList.size(value)
def elementCount(:stream, value), do: CacheStream.length(value)
def elementCount(:hash, value) when is_map(value), do: map_size(value)
def payloadBytes(:string, value), do: byte_size(value)
def payloadBytes(:list, value), do: sumBytes(value)
def payloadBytes(:set, value), do: value |> MapSet.to_list() |> sumBytes()
def payloadBytes(:zset, value) do
Enum.reduce(SkipList.to_list(value), 0, fn {_score, member}, total ->
total + byte_size(member) + wordSize()
end)
end
def payloadBytes(:stream, value) do
Enum.reduce(CacheStream.toList(value), 0, fn {_id, fields}, total ->
total + 2 * wordSize() + sumBytes(fields)
end)
end
def payloadBytes(:hash, value) when is_map(value) do
Enum.reduce(value, 0, fn {field, member}, total ->
total + byte_size(field) + byte_size(member)
end)
end
def describe(key, {kind, value, expiresAt}) do
total = estimateEntry(key, {kind, value, expiresAt})
payload = payloadBytes(kind, value)
%{
key: key,
kind: kind,
bytes: total,
keyBytes: keyBytes(key),
valueBytes: estimateValue(kind, value),
payloadBytes: payload,
overheadBytes: total - payload,
elements: elementCount(kind, value),
volatile: expiresAt != nil
}
end
def usage(key) do
case Store.watch_snapshot(key) do
:miss -> nil
entry -> estimateEntry(key, entry)
end
end
def describeKey(key) do
case Store.watch_snapshot(key) do
:miss -> nil
entry -> describe(key, entry)
end
end
def usedMemory, do: usedMemory(Store.dump())
def usedMemory(entries) when is_list(entries),
do: Enum.reduce(entries, 0, fn entry, total -> total + entryBytes(entry) end)
def usageByType, do: usageByType(Store.dump())
def usageByType(entries) when is_list(entries) do
empty = Map.new(@kinds, &{&1, %{keys: 0, bytes: 0}})
Enum.reduce(entries, empty, fn {_key, kind, _value, _expiresAt} = entry, acc ->
Map.update!(acc, kind, fn %{keys: keys, bytes: bytes} ->
%{keys: keys + 1, bytes: bytes + entryBytes(entry)}
end)
end)
end
def stats, do: stats(Store.dump())
def stats(entries) when is_list(entries) do
keys = length(entries)
used = usedMemory(entries)
overhead =
Enum.reduce(entries, keys * entryOverhead(), fn {key, _kind, _value, _expiresAt}, total ->
total + keyBytes(key)
end)
%{
keys: keys,
usedMemory: used,
overheadBytes: overhead,
datasetBytes: used - overhead,
bytesPerKey: if(keys == 0, do: 0, else: div(used, keys)),
volatileKeys:
Enum.count(entries, fn {_key, _kind, _value, expiresAt} -> expiresAt != nil end),
byType: usageByType(entries),
largestKey: largestKey(entries)
}
end
defp largestKey([]), do: nil
defp largestKey(entries) do
entries
|> Enum.map(fn {key, _kind, _value, _expiresAt} = entry -> {key, entryBytes(entry)} end)
|> Enum.max_by(fn {_key, bytes} -> bytes end)
end
defp termBytes(term), do: :erts_debug.size(term) * wordSize() + refcBytes(term)
defp sumBytes(binaries),
do: Enum.reduce(binaries, 0, fn binary, total -> total + byte_size(binary) end)
defp refcBytes(term) when is_binary(term) do
size = byte_size(term)
if size > @heapBinaryLimit, do: size, else: 0
end
defp refcBytes(term) when is_list(term),
do: Enum.reduce(term, 0, fn element, total -> total + refcBytes(element) end)
defp refcBytes(%_struct{} = term), do: term |> Map.from_struct() |> refcBytes()
defp refcBytes(term) when is_map(term) do
Enum.reduce(term, 0, fn {mapKey, mapValue}, total ->
total + refcBytes(mapKey) + refcBytes(mapValue)
end)
end
defp refcBytes(term) when is_tuple(term), do: term |> Tuple.to_list() |> refcBytes()
defp refcBytes(_term), do: 0
end
+25 -10
View File
@@ -11,7 +11,7 @@ defmodule Excache.Server.Connection do
@impl true
def init(socket) do
base = %{socket: socket, buffer: "", subscriptions: MapSet.new()}
base = %{socket: socket, buffer: "", subscriptions: MapSet.new(), blocked: nil}
{:ok, Map.merge(base, Transaction.initial_state())}
end
@@ -22,16 +22,16 @@ defmodule Excache.Server.Connection do
end
def handle_info({:tcp, socket, data}, state) do
{output, state} = consume(state.buffer <> data, %{state | buffer: ""})
unless output == [] do
:gen_tcp.send(socket, output)
end
state = pump(%{state | buffer: state.buffer <> data})
:inet.setopts(socket, active: :once)
{:noreply, state}
end
def handle_info({:excacheUnblocked, ref, reply}, %{blocked: ref} = state) do
:gen_tcp.send(state.socket, Encoder.encode(reply))
{:noreply, pump(%{state | blocked: nil})}
end
def handle_info({:tcp_closed, _socket}, state), do: {:stop, :normal, state}
def handle_info({:tcp_error, _socket, _reason}, state), do: {:stop, :normal, state}
@@ -41,6 +41,14 @@ defmodule Excache.Server.Connection do
{:noreply, state}
end
defp pump(%{blocked: nil} = state) do
{output, state} = consume(state.buffer, %{state | buffer: ""})
unless output == [], do: :gen_tcp.send(state.socket, output)
state
end
defp pump(state), do: state
defp consume(buffer, state) do
case Parser.parse_command(buffer) do
:incomplete ->
@@ -51,9 +59,16 @@ defmodule Excache.Server.Connection do
{:ok, args, rest} ->
{replies, state} = Command.execute(args, state)
encoded = Enum.map(replies, &Encoder.encode/1)
{more, state} = consume(rest, state)
{[encoded | more], state}
continue(replies, rest, state)
end
end
defp continue([{:blocked, ref}], rest, state),
do: {[], %{state | buffer: rest, blocked: ref}}
defp continue(replies, rest, state) do
encoded = Enum.map(replies, &Encoder.encode/1)
{more, state} = consume(rest, state)
{[encoded | more], state}
end
end
+102 -34
View File
@@ -2,6 +2,8 @@ defmodule Excache.Store do
@moduledoc false
alias Excache.{Bitmap, HyperLogLog}
alias Excache.Config
alias Excache.Eviction.Tracker
def shard_count, do: Application.get_env(:excache, :shard_count, 8)
@@ -9,7 +11,21 @@ defmodule Excache.Store do
defp shard_for(key), do: shard_name(rem(:erlang.phash2(key), shard_count()))
defp call_key(key, op), do: GenServer.call(shard_for(key), {:op, op})
defp call_key(key, op) do
trackAccess(key, op)
GenServer.call(shard_for(key), {:op, op})
end
defp trackAccess(_key, {:del, _target}), do: :ok
defp trackAccess(_key, {:peek, _target}), do: :ok
defp trackAccess(key, _op) do
if Config.tracksAccess?() do
Tracker.touchAsync(Tracker, key, System.system_time(:millisecond))
end
:ok
end
defp fan_out(op) do
for index <- 0..(shard_count() - 1) do
@@ -265,23 +281,37 @@ defmodule Excache.Store do
def touch(keys), do: exists(keys)
def hmget(key, fields) do
Enum.map(fields, fn field ->
collectPerField(fields, fn field ->
case call_key(key, {:hget, key, field}) do
{:ok, value} -> value
_ -> nil
{:ok, value} -> {:ok, value}
nil -> {:ok, nil}
error -> error
end
end)
end
def smismember(key, members) do
Enum.map(members, fn member ->
collectPerField(members, fn member ->
case call_key(key, {:sismember, key, member}) do
{:ok, 1} -> 1
_ -> 0
{:ok, present} -> {:ok, present}
error -> error
end
end)
end
defp collectPerField(entries, lookup) do
Enum.reduce_while(entries, {:ok, []}, fn entry, {:ok, acc} ->
case lookup.(entry) do
{:ok, value} -> {:cont, {:ok, [value | acc]}}
error -> {:halt, error}
end
end)
|> case do
{:ok, values} -> Enum.reverse(values)
error -> error
end
end
def sintercard(keys, limit) do
with {:ok, members} <- set_op(keys, &MapSet.intersection/2) do
count = length(members)
@@ -290,30 +320,64 @@ defmodule Excache.Store do
end
def smove(source, dest, member) do
case call_key(source, {:srem, source, [member]}) do
{:ok, 0} ->
{:ok, 0}
with :ok <- ensureKind(dest, "set") do
case call_key(source, {:srem, source, [member]}) do
{:ok, 0} ->
{:ok, 0}
{:ok, _} ->
call_key(dest, {:sadd, dest, [member]})
{:ok, 1}
{:ok, _removed} ->
case call_key(dest, {:sadd, dest, [member]}) do
{:ok, _added} ->
{:ok, 1}
error ->
error
error ->
call_key(source, {:sadd, source, [member]})
error
end
error ->
error
end
end
end
def rpoplpush(source, dest) do
case call_key(source, {:rpop, source, nil}) do
{:ok, value} ->
call_key(dest, {:lpush, dest, [value]})
{:ok, value}
with :ok <- ensureKind(dest, "list") do
case call_key(source, {:rpop, source, nil}) do
{:ok, value} ->
case call_key(dest, {:lpush, dest, [value]}) do
{:ok, _length} ->
{:ok, value}
nil ->
{:ok, nil}
error ->
call_key(source, {:rpush, source, [value]})
error
end
error ->
error
nil ->
{:ok, nil}
error ->
error
end
end
end
defp ensureKind(key, expected) do
case call_key(key, {:type, key}) do
"none" -> :ok
^expected -> :ok
_ -> {:error, :wrongtype}
end
end
defp putEntry(dest, {kind, value, ea}),
do: call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})
def rename(key, key) do
case call_key(key, {:exists, key}) do
1 -> {:ok, "OK"}
0 -> {:error, :nokey}
end
end
@@ -322,10 +386,13 @@ defmodule Excache.Store do
:miss ->
{:error, :nokey}
{kind, value, ea} ->
call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})
call_key(source, {:del, source})
{:ok, "OK"}
entry ->
if putEntry(dest, entry) == 1 do
call_key(source, {:del, source})
{:ok, "OK"}
else
{:error, :nokey}
end
end
end
@@ -340,10 +407,12 @@ defmodule Excache.Store do
{:ok, 0}
0 ->
{kind, value, ea} = entry
call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})
call_key(source, {:del, source})
{:ok, 1}
if putEntry(dest, entry) == 1 do
call_key(source, {:del, source})
{:ok, 1}
else
{:error, :nokey}
end
end
end
end
@@ -353,12 +422,11 @@ defmodule Excache.Store do
:miss ->
{:ok, 0}
{kind, value, ea} ->
entry ->
if not replace and call_key(dest, {:exists, dest}) == 1 do
{:ok, 0}
else
call_key(dest, {:del, dest})
{:ok, call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})}
{:ok, putEntry(dest, entry)}
end
end
end
+3 -2
View File
@@ -55,6 +55,9 @@ defmodule Excache.Store.Ops.Binary do
:wrongtype ->
{:error, :wrongtype}
{:ok, current, _ea, _exists} when value == "" ->
{:ok, byte_size(current)}
{:ok, current, ea, _exists} ->
updated = overwrite(current, offset, value)
store(table, key, updated, ea)
@@ -149,8 +152,6 @@ defmodule Excache.Store.Ops.Binary do
end
end
defp overwrite(current, offset, value) when value == "", do: current
defp overwrite(current, offset, value) do
size = byte_size(current)
padded = if size < offset, do: current <> :binary.copy(<<0>>, offset - size), else: current
+6 -13
View File
@@ -313,22 +313,15 @@ defmodule Excache.Store.Ops.Extra do
scanned =
if rank < 0, do: Enum.reverse(indexed), else: indexed
skip = abs(rank) - 1
limit = if count == 0, do: :all, else: count
matches = collectMatches(scanned, element, abs(rank) - 1, matchLimit(count), maxlen, 0, [])
collectMatches(scanned, element, skip, limit, maxlen, 0, [])
|> then(fn matches ->
if count == nil do
case matches do
[] -> nil
[first | _] -> first
end
else
matches
end
end)
if count == nil, do: List.first(matches), else: matches
end
defp matchLimit(nil), do: 1
defp matchLimit(0), do: :all
defp matchLimit(count), do: count
defp collectMatches([], _element, _skip, _limit, _maxlen, _compared, acc),
do: Enum.reverse(acc)
+444
View File
@@ -0,0 +1,444 @@
defmodule Excache.BlockingTest do
use Excache.Test.ServerCase, async: false
alias Excache.Blocking
describe "immediate path" do
test "BLPOP takes the head when the list already holds data", %{conn: conn} do
assert {2, conn} = Client.command(conn, ["RPUSH", "jobs", "first", "second"])
assert {["jobs", "first"], conn} = Client.command(conn, ["BLPOP", "jobs", "0"])
assert {["second"], conn} = Client.command(conn, ["LRANGE", "jobs", "0", "-1"])
assert {["jobs", "second"], conn} = Client.command(conn, ["BLPOP", "jobs", "0"])
assert {0, _conn} = Client.command(conn, ["EXISTS", "jobs"])
end
test "BRPOP takes the tail", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["RPUSH", "tail", "a", "b", "c"])
assert {["tail", "c"], _conn} = Client.command(conn, ["BRPOP", "tail", "0"])
end
test "BLPOP serves the first key given that has data", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["RPUSH", "second", "later"])
assert {1, conn} = Client.command(conn, ["RPUSH", "first", "sooner"])
assert {["first", "sooner"], conn} =
Client.command(conn, ["BLPOP", "first", "second", "0"])
assert {["second", "later"], _conn} =
Client.command(conn, ["BLPOP", "first", "second", "0"])
end
test "BLPOP on a key holding another type answers WRONGTYPE without blocking", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "notalist", "v"])
assert {{:error, "WRONGTYPE" <> _}, _conn} =
Client.command(conn, ["BLPOP", "notalist", "0"])
end
test "argument errors", %{conn: conn} do
assert {{:error, "ERR timeout is negative"}, conn} =
Client.command(conn, ["BLPOP", "k", "-1"])
assert {{:error, "ERR timeout is not a float or out of range"}, conn} =
Client.command(conn, ["BLPOP", "k", "later"])
assert {{:error, "ERR wrong number of arguments for 'blpop' command"}, conn} =
Client.command(conn, ["BLPOP", "k"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["BLMOVE", "a", "b", "LEFT", "SIDEWAYS", "0"])
end
end
describe "blocking path" do
test "BLPOP waits until another connection pushes", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BLPOP", "queue", "0"])
awaitWaiters("queue", 1)
assert {1, conn} = Client.command(conn, ["LPUSH", "queue", "payload"])
assert Task.await(waiter) == ["queue", "payload"]
assert {0, _conn} = Client.command(conn, ["EXISTS", "queue"])
end
test "BRPOP waits on the tail side", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BRPOP", "queue", "0"])
awaitWaiters("queue", 1)
assert {2, _conn} = Client.command(conn, ["RPUSH", "queue", "old", "new"])
assert Task.await(waiter) == ["queue", "new"]
end
test "a waiter registered on several keys wakes on whichever key receives data", %{
conn: conn,
port: port
} do
waiter = blockingTask(port, ["BLPOP", "left", "right", "0"])
awaitWaiters("right", 1)
assert {1, _conn} = Client.command(conn, ["LPUSH", "right", "hello"])
assert Task.await(waiter) == ["right", "hello"]
end
test "a timed out BLPOP answers with a nil array on the wire", %{port: port} do
socket = rawConnect(port)
:ok = :gen_tcp.send(socket, encodeCommand(["BLPOP", "never", "0.15"]))
{elapsed, reply} = :timer.tc(fn -> :gen_tcp.recv(socket, 0, 2000) end)
assert reply == {:ok, "*-1\r\n"}
assert elapsed > 100_000
assert elapsed < 1_500_000
:gen_tcp.close(socket)
end
test "a blocked connection still answers commands sent after it is unblocked", %{
conn: conn,
port: port
} do
pipelined = Client.connect(port)
pipelined =
Client.send_raw(pipelined, [
encodeCommand(["BLPOP", "pipeq", "0"]),
encodeCommand(["PING"])
])
awaitWaiters("pipeq", 1)
assert :gen_tcp.recv(pipelined.socket, 0, 100) == {:error, :timeout}
assert {1, _conn} = Client.command(conn, ["LPUSH", "pipeq", "v"])
assert {["pipeq", "v"], pipelined} = Client.read(pipelined)
assert {{:simple, "PONG"}, pipelined} = Client.read(pipelined)
Client.close(pipelined)
end
end
describe "delivery accounting" do
test "three waiters are served in the order they blocked", %{conn: conn, port: port} do
first = blockingTask(port, ["BLPOP", "fifo", "0"])
awaitWaiters("fifo", 1)
second = blockingTask(port, ["BLPOP", "fifo", "0"])
awaitWaiters("fifo", 2)
third = blockingTask(port, ["BLPOP", "fifo", "0"])
awaitWaiters("fifo", 3)
assert {1, conn} = Client.command(conn, ["LPUSH", "fifo", "a"])
assert Task.await(first) == ["fifo", "a"]
assert {1, conn} = Client.command(conn, ["LPUSH", "fifo", "b"])
assert Task.await(second) == ["fifo", "b"]
assert {1, _conn} = Client.command(conn, ["LPUSH", "fifo", "c"])
assert Task.await(third) == ["fifo", "c"]
end
test "one pushed element wakes exactly one of two waiters", %{conn: conn, port: port} do
first = blockingTask(port, ["BLPOP", "single", "0"])
awaitWaiters("single", 1)
second = blockingTask(port, ["BLPOP", "single", "0"])
awaitWaiters("single", 2)
assert {1, conn} = Client.command(conn, ["LPUSH", "single", "only"])
assert Task.await(first) == ["single", "only"]
assert Task.yield(second, 150) == nil
assert Blocking.waitingCount("single") == 1
assert {0, conn} = Client.command(conn, ["EXISTS", "single"])
assert {1, _conn} = Client.command(conn, ["LPUSH", "single", "later"])
assert Task.await(second) == ["single", "later"]
end
test "a two element push wakes both waiters with one element each", %{
conn: conn,
port: port
} do
first = blockingTask(port, ["BLPOP", "pair", "0"])
awaitWaiters("pair", 1)
second = blockingTask(port, ["BLPOP", "pair", "0"])
awaitWaiters("pair", 2)
assert {2, conn} = Client.command(conn, ["LPUSH", "pair", "a", "b"])
assert Task.await(first) == ["pair", "b"]
assert Task.await(second) == ["pair", "a"]
assert {0, _conn} = Client.command(conn, ["EXISTS", "pair"])
end
test "a dropped connection leaves the element for the next client", %{
conn: conn,
port: port
} do
victim = Client.connect(port)
Client.send_raw(victim, encodeCommand(["BLPOP", "orphan", "0"]))
awaitWaiters("orphan", 1)
Client.close(victim)
awaitUntil(fn -> Blocking.waitingCount("orphan") == 0 end)
assert {1, conn} = Client.command(conn, ["LPUSH", "orphan", "kept"])
assert {["kept"], conn} = Client.command(conn, ["LRANGE", "orphan", "0", "-1"])
assert {["orphan", "kept"], _conn} = Client.command(conn, ["BLPOP", "orphan", "0"])
end
end
describe "isolation" do
test "a blocked client stalls neither other clients nor the expiry sweeper", %{
conn: conn,
port: port
} do
subscriber = Client.connect(port)
assert {["subscribe", "__keyevent@0__:expired", 1], subscriber} =
Client.command(subscriber, ["SUBSCRIBE", "__keyevent@0__:expired"])
waiter = blockingTask(port, ["BLPOP", "idle", "1"])
awaitWaiters("idle", 1)
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "other", "v"])
assert {"v", conn} = Client.command(conn, ["GET", "other"])
assert {3, conn} = Client.command(conn, ["RPUSH", "unrelated", "a", "b", "c"])
assert {["a", "b", "c"], conn} = Client.command(conn, ["LRANGE", "unrelated", "0", "-1"])
assert {{:simple, "OK"}, _conn} =
Client.command(conn, ["SET", "ephemeral", "v", "PX", "60"])
assert {["message", "__keyevent@0__:expired", "ephemeral"], _subscriber} =
Client.read(subscriber)
assert Task.await(waiter) == nil
Client.close(subscriber)
end
test "blocking commands inside MULTI answer at once instead of waiting", %{port: port} do
socket = rawConnect(port)
:ok =
:gen_tcp.send(socket, [
encodeCommand(["MULTI"]),
encodeCommand(["BLPOP", "txn", "0"]),
encodeCommand(["EXEC"])
])
assert readBytes(socket, byte_size("+OK\r\n+QUEUED\r\n*1\r\n*-1\r\n")) ==
"+OK\r\n+QUEUED\r\n*1\r\n*-1\r\n"
:gen_tcp.close(socket)
end
test "a queued blocking command still pops available data on EXEC", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["RPUSH", "txn", "ready"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["BLPOP", "txn", "0"])
assert {[["txn", "ready"]], conn} = Client.command(conn, ["EXEC"])
assert {0, _conn} = Client.command(conn, ["EXISTS", "txn"])
end
test "a push made inside a transaction wakes a waiter", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BLPOP", "txnpush", "0"])
awaitWaiters("txnpush", 1)
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["LPUSH", "txnpush", "fromtxn"])
assert {[1], _conn} = Client.command(conn, ["EXEC"])
assert Task.await(waiter) == ["txnpush", "fromtxn"]
end
end
describe "persistence and notifications" do
test "a woken BLPOP is logged as LPOP and replays without blocking", %{
conn: conn,
port: port
} do
aofPath = Application.get_env(:excache, :aof_path)
waiter = blockingTask(port, ["BLPOP", "aofq", "0"])
awaitWaiters("aofq", 1)
assert {1, conn} = Client.command(conn, ["LPUSH", "aofq", "payload"])
assert Task.await(waiter) == ["aofq", "payload"]
Excache.Aof.flush()
log = File.read!(aofPath)
assert log =~ "LPOP"
refute log =~ "BLPOP"
Excache.Store.flushdb()
Excache.Aof.replay(aofPath)
assert {0, _conn} = Client.command(conn, ["EXISTS", "aofq"])
end
test "a woken BLPOP fires the same keyspace event as LPOP", %{conn: conn, port: port} do
subscriber = Client.connect(port)
assert {["subscribe", "__keyevent@0__:lpop", 1], subscriber} =
Client.command(subscriber, ["SUBSCRIBE", "__keyevent@0__:lpop"])
waiter = blockingTask(port, ["BLPOP", "notified", "0"])
awaitWaiters("notified", 1)
assert {1, _conn} = Client.command(conn, ["LPUSH", "notified", "v"])
assert Task.await(waiter) == ["notified", "v"]
assert {["message", "__keyevent@0__:lpop", "notified"], _subscriber} =
Client.read(subscriber)
Client.close(subscriber)
end
end
describe "BLMOVE and BRPOPLPUSH" do
test "BRPOPLPUSH moves an available element straight away", %{conn: conn} do
assert {2, conn} = Client.command(conn, ["RPUSH", "src", "a", "b"])
assert {"b", conn} = Client.command(conn, ["BRPOPLPUSH", "src", "dst", "0"])
assert {["b"], conn} = Client.command(conn, ["LRANGE", "dst", "0", "-1"])
assert {["a"], _conn} = Client.command(conn, ["LRANGE", "src", "0", "-1"])
end
test "BRPOPLPUSH waits for the source and times out with nil", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BRPOPLPUSH", "src", "dst", "0"])
awaitWaiters("src", 1)
assert {1, conn} = Client.command(conn, ["RPUSH", "src", "moved"])
assert Task.await(waiter) == "moved"
assert {["moved"], conn} = Client.command(conn, ["LRANGE", "dst", "0", "-1"])
assert {nil, _conn} = Client.command(conn, ["BRPOPLPUSH", "empty", "dst", "0.1"])
end
test "BLMOVE honours both sides while blocked", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BLMOVE", "src", "dst", "LEFT", "RIGHT", "0"])
awaitWaiters("src", 1)
assert {2, conn} = Client.command(conn, ["RPUSH", "src", "head", "tail"])
assert Task.await(waiter) == "head"
assert {["tail"], conn} = Client.command(conn, ["LRANGE", "src", "0", "-1"])
assert {2, conn} = Client.command(conn, ["RPUSH", "dst", "existing"])
assert {"tail", conn} = Client.command(conn, ["BLMOVE", "src", "dst", "RIGHT", "LEFT", "0"])
assert {["tail", "head", "existing"], _conn} =
Client.command(conn, ["LRANGE", "dst", "0", "-1"])
end
test "a BLMOVE destination wakes a client blocked on it", %{conn: conn, port: port} do
consumer = blockingTask(port, ["BLPOP", "stage2", "0"])
awaitWaiters("stage2", 1)
mover = blockingTask(port, ["BLMOVE", "stage1", "stage2", "LEFT", "RIGHT", "0"])
awaitWaiters("stage1", 1)
assert {1, conn} = Client.command(conn, ["LPUSH", "stage1", "item"])
assert Task.await(mover) == "item"
assert Task.await(consumer) == ["stage2", "item"]
assert {0, _conn} = Client.command(conn, ["EXISTS", "stage2"])
end
test "a rename that replaces a waited key with another type reports WRONGTYPE", %{
conn: conn,
port: port
} do
waiter = blockingTask(port, ["BLPOP", "renamed", "0"])
awaitWaiters("renamed", 1)
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", "v"])
assert {{:simple, "OK"}, _conn} = Client.command(conn, ["RENAME", "plain", "renamed"])
assert {:error, "WRONGTYPE" <> _} = Task.await(waiter)
end
end
describe "BLMPOP" do
test "pops up to COUNT elements from the first non empty key", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["RPUSH", "mp", "a", "b", "c"])
assert {["mp", ["a", "b"]], conn} =
Client.command(conn, ["BLMPOP", "0", "2", "missing", "mp", "LEFT", "COUNT", "2"])
assert {["mp", ["c"]], _conn} =
Client.command(conn, ["BLMPOP", "0", "1", "mp", "LEFT"])
end
test "blocks across several keys and answers RIGHT side", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BLMPOP", "0", "2", "mp1", "mp2", "RIGHT", "COUNT", "2"])
awaitWaiters("mp2", 1)
assert {3, conn} = Client.command(conn, ["RPUSH", "mp2", "x", "y", "z"])
assert Task.await(waiter) == ["mp2", ["z", "y"]]
assert {["x"], _conn} = Client.command(conn, ["LRANGE", "mp2", "0", "-1"])
end
test "times out with a nil array", %{conn: conn} do
assert {nil, _conn} = Client.command(conn, ["BLMPOP", "0.1", "1", "mp", "LEFT"])
end
test "argument errors", %{conn: conn} do
assert {{:error, "ERR numkeys should be greater than 0"}, conn} =
Client.command(conn, ["BLMPOP", "0", "0", "mp", "LEFT"])
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["BLMPOP", "0", "1", "mp", "SIDEWAYS"])
assert {{:error, "ERR count should be greater than 0"}, conn} =
Client.command(conn, ["BLMPOP", "0", "1", "mp", "LEFT", "COUNT", "0"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["BLMPOP", "0", "2", "mp", "LEFT"])
end
end
defp blockingTask(port, args) do
Task.async(fn ->
client = Client.connect(port)
{reply, _client} = Client.command(client, args)
Client.close(client)
reply
end)
end
defp awaitWaiters(key, count) do
awaitUntil(fn -> Blocking.waitingCount(key) == count end)
end
defp awaitUntil(check, attempts \\ 400) do
cond do
check.() ->
:ok
attempts == 0 ->
flunk("the expected registry state was never reached")
true ->
Process.sleep(5)
awaitUntil(check, attempts - 1)
end
end
defp rawConnect(port) do
{:ok, socket} = :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false, packet: :raw])
socket
end
defp readBytes(socket, size) do
{:ok, bytes} = :gen_tcp.recv(socket, size, 2000)
bytes
end
defp encodeCommand(args) do
[
"*",
Integer.to_string(length(args)),
"\r\n",
Enum.map(args, fn arg -> ["$", Integer.to_string(byte_size(arg)), "\r\n", arg, "\r\n"] end)
]
end
end
+50
View File
@@ -0,0 +1,50 @@
defmodule Excache.CommandHandshakeTest do
@moduledoc """
Client libraries ask the server what it can do before they send anything else.
Until the catalog was wired in, `COMMAND` answered with an empty array over the
wire - a server that claims to support nothing. These tests pin the answer to
the real dispatch table, over a real socket.
"""
use Excache.Test.ServerCase, async: false
test "COMMAND reports the whole dispatch table over the wire", %{conn: conn} do
{entries, conn} = Client.command(conn, ["COMMAND"])
assert is_list(entries)
assert length(entries) == length(Enum.uniq(Excache.Command.known_commands()))
names =
entries
|> Enum.map(fn [name | _rest] -> String.upcase(name) end)
|> MapSet.new()
for command <- ["GET", "SET", "BLPOP", "MEMORY", "CONFIG", "SCAN", "XADD"] do
assert MapSet.member?(names, command), "#{command} missing from COMMAND"
end
{count, _conn} = Client.command(conn, ["COMMAND", "COUNT"])
assert count == length(entries)
end
test "COMMAND INFO carries the arity and key positions a router needs", %{conn: conn} do
{[["get", arity, flags, firstKey, lastKey, step]], conn} =
Client.command(conn, ["COMMAND", "INFO", "GET"])
assert arity == 2
assert firstKey == 1
assert lastKey == 1
assert step == 1
assert is_list(flags)
{[entry], _conn} = Client.command(conn, ["COMMAND", "INFO", "nosuchcommand"])
assert entry == nil
end
test "COMMAND GETKEYS extracts the keys a command touches", %{conn: conn} do
assert {["one", "two"], conn} =
Client.command(conn, ["COMMAND", "GETKEYS", "MSET", "one", "1", "two", "2"])
assert {["only"], _conn} = Client.command(conn, ["COMMAND", "GETKEYS", "GET", "only"])
end
end
+160
View File
@@ -0,0 +1,160 @@
defmodule Excache.ConfigTest do
use ExUnit.Case, async: false
alias Excache.Config
alias Excache.Eviction
alias Excache.Eviction.Tracker
setup do
Config.reset()
on_exit(&Config.reset/0)
:ok
end
test "maxmemory accepts the Redis unit suffixes and refuses everything else" do
sizes = [
{"0", 0},
{"100", 100},
{"512b", 512},
{"1k", 1000},
{"1kb", 1024},
{"1m", 1_000_000},
{"8MB", 8 * 1024 * 1024},
{"1g", 1_000_000_000},
{"2gb", 2 * 1024 * 1024 * 1024}
]
for {raw, bytes} <- sizes do
assert Config.set("maxmemory", raw) == :ok
assert Config.maxmemory() == bytes
end
for raw <- ["", "abc", "-1", "1 mb", "1tb", "kb", "1.5mb", "1kbb", "mb1"] do
assert {:error, {:invalid, hint}} = Config.set("maxmemory", raw)
assert hint =~ "memory value"
end
assert Config.maxmemory() == 2 * 1024 * 1024 * 1024
end
test "the default policy is noeviction and every Redis spelling round trips" do
assert Config.policy() == :noeviction
assert Config.get("maxmemory-policy") == {:ok, :noeviction}
for policy <- Eviction.policies() do
name = Eviction.policyName(policy)
assert Config.set("maxmemory-policy", name) == :ok
assert Config.policy() == policy
assert Config.getGlob("maxmemory-policy") == [{"maxmemory-policy", name}]
end
assert {:error, {:invalid, hint}} = Config.set("maxmemory-policy", "lru")
assert hint =~ "allkeys-lru"
assert Config.policy() == :volatileRandom
end
test "only the lru and lfu policies switch access tracking on" do
refute Config.tracksAccess?()
for name <- ~w(allkeys-lru allkeys-lfu volatile-lru volatile-lfu) do
assert Config.set("maxmemory-policy", name) == :ok
assert Config.tracksAccess?()
end
for name <- ~w(noeviction allkeys-random volatile-random volatile-ttl) do
assert Config.set("maxmemory-policy", name) == :ok
refute Config.tracksAccess?()
end
end
test "integer parameters are range checked and keep the last valid value" do
assert Config.maxmemorySamples() == Eviction.defaultSampleSize()
assert Config.set("maxmemory-samples", "1") == :ok
assert Config.set("maxmemory-samples", "64") == :ok
assert Config.maxmemorySamples() == 64
for raw <- ["0", "65", "ten", "-3", ""] do
assert {:error, {:invalid, hint}} = Config.set("maxmemory-samples", raw)
assert hint == "argument must be an integer between 1 and 64"
end
assert Config.maxmemorySamples() == 64
end
test "the lfu knobs reach the running tracker and come back on reset" do
assert Tracker.settings(Tracker) == %{logFactor: 10, decayTime: 1}
assert Config.set("lfu-log-factor", "42") == :ok
assert Config.set("lfu-decay-time", "7") == :ok
assert Config.lfuLogFactor() == 42
assert Config.lfuDecayTime() == 7
assert Tracker.settings(Tracker) == %{logFactor: 42, decayTime: 7}
Config.reset()
assert Tracker.settings(Tracker) == %{logFactor: 10, decayTime: 1}
end
test "port and the aof path are readable but cannot be changed at runtime" do
assert Config.get("port") == {:ok, Application.get_env(:excache, :port)}
assert Config.set("port", "6380") == {:error, :immutable}
assert Config.get("aof_path") == {:ok, Excache.Aof.path()}
assert Config.set("aof_path", "/tmp/excache-should-not-move.aof") == {:error, :immutable}
assert Config.get("aof_path") == {:ok, Excache.Aof.path()}
end
test "the snapshot path is settable and persistence follows it" do
previous = Application.get_env(:excache, :snapshot_path)
on_exit(fn -> Application.put_env(:excache, :snapshot_path, previous) end)
target = Path.join(System.tmp_dir!(), "excache-config-test.snapshot")
assert Config.set("snapshot_path", target) == :ok
assert Excache.Persistence.snapshot_path() == target
assert Config.get("snapshot_path") == {:ok, target}
assert {:error, {:invalid, _hint}} = Config.set("snapshot_path", "")
assert Excache.Persistence.snapshot_path() == target
end
test "unknown parameters are refused on read, write and glob" do
assert Config.get("nosuch") == :error
assert Config.set("nosuch", "1") == {:error, :unknown}
assert Config.getGlob("nosuch*") == []
end
test "getGlob follows glob semantics and answers sorted, formatted pairs" do
assert Config.getGlob("maxmemory") == [{"maxmemory", "0"}]
assert Config.getGlob("maxmemory*") == [
{"maxmemory", "0"},
{"maxmemory-policy", "noeviction"},
{"maxmemory-samples", "5"}
]
assert Config.getGlob("lfu-*") == [{"lfu-decay-time", "1"}, {"lfu-log-factor", "10"}]
assert Config.getGlob("?axmemory") == [{"maxmemory", "0"}]
assert Config.getGlob("*policy") == [{"maxmemory-policy", "noeviction"}]
assert Config.getGlob("MAXMEMORY") == [{"maxmemory", "0"}]
assert Config.getGlob("maxmemor") == []
assert Enum.map(Config.getGlob("*"), fn {name, _value} -> name end) == Config.names()
end
test "parameter names are case insensitive" do
assert Config.set("MaxMemory", "2kb") == :ok
assert Config.get("MAXMEMORY") == {:ok, 2048}
assert Config.getGlob("maxmemory") == [{"maxmemory", "2048"}]
end
test "setMany applies every pair or leaves all of them alone" do
assert Config.setMany([{"maxmemory", "4kb"}, {"maxmemory-policy", "allkeys-lfu"}]) == :ok
assert Config.maxmemory() == 4096
assert Config.policy() == :allkeysLfu
assert {:error, "maxmemory-policy", {:invalid, _hint}} =
Config.setMany([{"maxmemory", "8kb"}, {"maxmemory-policy", "nope"}])
assert Config.maxmemory() == 4096
assert Config.policy() == :allkeysLfu
end
end
+202
View File
@@ -0,0 +1,202 @@
defmodule Excache.EvictionRuntimeTest do
use Excache.Test.ServerCase, async: false
alias Excache.Aof
alias Excache.Config
alias Excache.Eviction.Runner
alias Excache.Eviction.Tracker
alias Excache.Store
@payload String.duplicate("x", 1000)
@oom {:error, "OOM command not allowed when used memory > 'maxmemory'"}
setup_all do
Application.put_env(:excache, :eviction_interval, 60_000)
Process.sleep(250)
on_exit(fn -> Application.delete_env(:excache, :eviction_interval) end)
:ok
end
setup do
Application.put_env(:excache, :eviction_recheck, 0)
Config.reset()
Tracker.clear(Tracker)
Runner.resetStats()
on_exit(fn ->
Config.reset()
Application.delete_env(:excache, :eviction_recheck)
end)
:ok
end
test "allkeys-lru drops the coldest keys when a write crosses maxmemory", %{conn: conn} do
conn = configure(conn, ["maxmemory-policy", "allkeys-lru", "maxmemory-samples", "64"])
conn = fill(conn, 1..40)
{before, conn} = stats(conn)
assert before["keys.count"] == 40
assert before["total.allocated"] > 40_000
Process.sleep(5)
conn = touch(conn, 1..10)
limit = div(before["total.allocated"], 2)
conn = configure(conn, ["maxmemory", Integer.to_string(limit)])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "trigger", @payload])
{evicted, conn} = stats(conn)
freed = evicted["evicted.keys"]
assert freed >= 15 and freed <= 25
assert evicted["keys.count"] == 41 - freed
assert evicted["total.allocated"] < before["total.allocated"]
assert evicted["total.allocated"] <= limit + 2 * before["keys.bytes-per-key"]
assert evicted["oom.rejections"] == 0
assert Store.dbsize() == evicted["keys.count"]
assert Tracker.size(Tracker) == evicted["keys.count"]
assert {10, conn} = Client.command(conn, ["EXISTS" | keys(1..10)])
assert {cold, conn} = Client.command(conn, ["EXISTS" | keys(11..40)])
assert cold == 30 - freed
Aof.flush()
deletes = Aof.path() |> File.read!() |> String.split("$3\r\nDEL\r\n") |> length()
assert deletes - 1 == freed
{doctor, _conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
assert doctor =~ "#{freed} keys have been evicted"
end
test "noeviction refuses growing writes but keeps freeing ones", %{conn: conn} do
conn = fill(conn, 1..30)
{before, conn} = stats(conn)
limit = div(before["total.allocated"], 2)
conn = configure(conn, ["maxmemory", Integer.to_string(limit)])
assert {@oom, conn} = Client.command(conn, ["SET", "blocked", @payload])
assert {0, conn} = Client.command(conn, ["EXISTS", "blocked"])
assert {@oom, conn} = Client.command(conn, ["APPEND", "key:1", "more"])
assert {@oom, conn} = Client.command(conn, ["RPUSH", "fresh-list", "value"])
assert {@payload, conn} = Client.command(conn, ["GET", "key:1"])
assert {30, conn} = Client.command(conn, ["DBSIZE"])
{blocked, conn} = stats(conn)
assert blocked["evicted.keys"] == 0
assert blocked["oom.rejections"] >= 3
assert blocked["total.allocated"] == before["total.allocated"]
assert {20, conn} = Client.command(conn, ["DEL" | keys(1..20)])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "allowed", @payload])
assert {11, conn} = Client.command(conn, ["DBSIZE"])
{recovered, _conn} = stats(conn)
assert recovered["total.allocated"] < limit
end
test "maxmemory 0 lifts the limit again", %{conn: conn} do
conn = fill(conn, 1..20)
{before, conn} = stats(conn)
conn = configure(conn, ["maxmemory", Integer.to_string(div(before["total.allocated"], 2))])
assert {@oom, conn} = Client.command(conn, ["SET", "blocked", @payload])
conn = configure(conn, ["maxmemory", "0"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "blocked", @payload])
assert {21, _conn} = Client.command(conn, ["DBSIZE"])
end
test "a volatile policy with no TTL anywhere cannot free a thing", %{conn: conn} do
conn = configure(conn, ["maxmemory-policy", "volatile-lru", "maxmemory-samples", "64"])
conn = fill(conn, 1..20)
{before, conn} = stats(conn)
limit = div(before["total.allocated"], 2)
conn = configure(conn, ["maxmemory", Integer.to_string(limit)])
assert {@oom, conn} = Client.command(conn, ["SET", "blocked", @payload])
assert {20, conn} = Client.command(conn, ["DBSIZE"])
conn =
Enum.reduce(1..15, conn, fn index, acc ->
{1, acc} = Client.command(acc, ["EXPIRE", "key:#{index}", "600"])
acc
end)
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "allowed", @payload])
{after_, conn} = stats(conn)
assert after_["evicted.keys"] >= 10
assert after_["keys.count"] == 21 - after_["evicted.keys"]
assert {5, _conn} = Client.command(conn, ["EXISTS" | keys(16..20)])
end
test "a read refreshes access metadata only while a tracking policy is set", %{conn: conn} do
conn = fill(conn, 1..2)
assert Tracker.size(Tracker) == 0
conn = configure(conn, ["maxmemory-policy", "allkeys-lru"])
conn = fill(conn, 1..2)
assert Tracker.size(Tracker) == 2
Process.sleep(5)
conn = touch(conn, 1..1)
now = System.system_time(:millisecond)
assert Tracker.idleMs(Tracker, "key:1", now) < Tracker.idleMs(Tracker, "key:2", now)
conn = configure(conn, ["maxmemory-policy", "noeviction"])
Tracker.clear(Tracker)
_conn = touch(conn, 1..2)
assert Tracker.size(Tracker) == 0
end
test "a sweep drops metadata for keys that are gone", %{conn: conn} do
conn = configure(conn, ["maxmemory-policy", "allkeys-lru"])
conn = fill(conn, 1..10)
assert Tracker.size(Tracker) == 10
assert {5, conn} = Client.command(conn, ["DEL" | keys(1..5)])
assert Tracker.size(Tracker) == 10
conn = configure(conn, ["maxmemory", "1gb"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "fresh", "value"])
assert Store.dbsize() == 6
assert Tracker.size(Tracker) == 6
{reported, _conn} = stats(conn)
assert reported["tracker.keys"] == 6
assert reported["evicted.keys"] == 0
end
defp configure(conn, args) do
{{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET" | args])
conn
end
defp fill(conn, range) do
Enum.reduce(range, conn, fn index, acc ->
{{:simple, "OK"}, acc} = Client.command(acc, ["SET", "key:#{index}", @payload])
acc
end)
end
defp touch(conn, range) do
Enum.reduce(range, conn, fn index, acc ->
{@payload, acc} = Client.command(acc, ["GET", "key:#{index}"])
acc
end)
end
defp keys(range), do: Enum.map(range, &"key:#{&1}")
defp stats(conn) do
{flat, conn} = Client.command(conn, ["MEMORY", "STATS"])
{flat |> Enum.chunk_every(2) |> Map.new(fn [name, value] -> {name, value} end), conn}
end
end
+380
View File
@@ -0,0 +1,380 @@
defmodule Excache.EvictionTest do
use ExUnit.Case, async: false
alias Excache.Eviction
alias Excache.Eviction.Tracker
alias Excache.Memory
alias Excache.Store
@now 1_000_000
setup do
Store.flushdb()
on_exit(&Store.flushdb/0)
:ok
end
test "policy names round trip through the Redis spelling" do
for policy <- Eviction.policies() do
assert {:ok, ^policy} = Eviction.parsePolicy(Eviction.policyName(policy))
end
assert Enum.sort(Enum.map(Eviction.policies(), &Eviction.policyName/1)) ==
Enum.sort(~w(
noeviction allkeys-lru allkeys-lfu allkeys-random
volatile-lru volatile-lfu volatile-ttl volatile-random
))
assert Eviction.parsePolicy("lru") == :error
assert Eviction.volatile?(:volatileTtl)
assert Eviction.allkeys?(:allkeysLfu)
assert Eviction.random?(:volatileRandom)
refute Eviction.volatile?(:allkeysLru)
end
test "every policy picks a different victim from the same keyspace" do
opts = [now: @now, metadata: mixedMetadata(), sampleSize: 8, sampler: firstSampler()]
entries = mixedEntries()
assert Eviction.selectVictim(entries, [policy: :allkeysLru] ++ opts) == {:ok, "idle-all"}
assert Eviction.selectVictim(entries, [policy: :allkeysLfu] ++ opts) == {:ok, "cold-all"}
assert Eviction.selectVictim(entries, [policy: :allkeysRandom] ++ opts) == {:ok, "random-all"}
assert Eviction.selectVictim(entries, [policy: :volatileLru] ++ opts) == {:ok, "idle-ttl"}
assert Eviction.selectVictim(entries, [policy: :volatileLfu] ++ opts) == {:ok, "cold-ttl"}
assert Eviction.selectVictim(entries, [policy: :volatileTtl] ++ opts) == {:ok, "soon-ttl"}
assert Eviction.selectVictim(entries, [policy: :volatileRandom] ++ opts) ==
{:ok, "random-ttl"}
assert Eviction.selectVictim(entries, [policy: :noeviction] ++ opts) ==
{:error, :noCandidates}
victims =
for policy <- Eviction.policies() -- [:noeviction] do
{:ok, key} = Eviction.selectVictim(entries, [policy: policy] ++ opts)
key
end
assert length(Enum.uniq(victims)) == 7
end
test "an untracked key is evicted before any tracked one" do
entries = stringEntries(["tracked", "unseen"], "payload")
metadata = %{"tracked" => %{lastAccess: @now - 500_000, counter: 5}}
opts = [now: @now, metadata: metadata, sampleSize: 8, sampler: firstSampler()]
assert Eviction.selectVictim(entries, [policy: :allkeysLru] ++ opts) == {:ok, "unseen"}
assert Eviction.selectVictim(entries, [policy: :allkeysLfu] ++ opts) == {:ok, "unseen"}
end
test "noeviction refuses to free anything and reports OOM" do
entries = stringEntries(~w(k1 k2 k3), "payload")
{table, keyspace} = fixtureKeyspace(entries)
assert {:error, report} =
Eviction.run(policy: :noeviction, maxmemory: 1, keyspace: keyspace, now: @now)
assert report.reason == :noeviction
assert report.evicted == []
assert report.evictedCount == 0
assert report.freedBytes == 0
assert report.usedMemory == Memory.usedMemory(entries)
assert :ets.info(table, :size) == 3
end
test "volatile policies leave a keyspace without any TTL untouched" do
entries = stringEntries(~w(k1 k2 k3), "payload")
{table, keyspace} = fixtureKeyspace(entries)
for policy <- [:volatileLru, :volatileLfu, :volatileTtl, :volatileRandom] do
assert {:error, report} =
Eviction.run(
policy: policy,
maxmemory: 1,
keyspace: keyspace,
now: @now,
metadata: %{}
)
assert report.reason == :noVolatileKeys
assert report.evictedCount == 0
assert report.freedBytes == 0
end
assert :ets.info(table, :size) == 3
end
test "maxmemory of zero means no limit at all" do
entries = stringEntries(~w(k1 k2 k3), "payload")
{table, keyspace} = fixtureKeyspace(entries)
assert {:ok, report} =
Eviction.run(policy: :allkeysRandom, maxmemory: 0, keyspace: keyspace, now: @now)
assert report.reason == :withinLimit
assert report.evicted == []
assert :ets.info(table, :size) == 3
end
test "the loop stops the moment usage drops under the limit" do
keys = for index <- 1..10, do: "k#{String.pad_leading("#{index}", 2, "0")}"
entries = stringEntries(keys, "payload")
perEntry = Memory.entryBytes(hd(entries))
{table, keyspace} = fixtureKeyspace(entries)
assert {:ok, report} =
Eviction.run(
policy: :allkeysRandom,
maxmemory: 7 * perEntry,
keyspace: keyspace,
now: @now,
seed: {2, 4, 8}
)
assert report.reason == :evicted
assert report.evictedCount == 3
assert report.freedBytes == 3 * perEntry
assert report.usedMemory == 7 * perEntry
assert report.passes == 3
assert :ets.info(table, :size) == 7
assert Enum.all?(report.evicted, fn key -> :ets.lookup(table, key) == [] end)
end
test "the candidate pool carries a strong candidate between passes" do
entries = stringEntries(~w(k1 k2 k3 k4), "payload")
perEntry = Memory.entryBytes(hd(entries))
{_table, keyspace} = fixtureKeyspace(entries)
metadata = %{
"k1" => %{lastAccess: @now - 90_000, counter: 9},
"k2" => %{lastAccess: @now - 80_000, counter: 9},
"k3" => %{lastAccess: @now - 20_000, counter: 9},
"k4" => %{lastAccess: @now - 30_000, counter: 9}
}
assert {:ok, report} =
Eviction.run(
policy: :allkeysLru,
maxmemory: 2 * perEntry,
keyspace: keyspace,
now: @now,
metadata: metadata,
sampleSize: 2,
sampler: windowSampler()
)
assert report.evicted == ["k1", "k2"]
end
test "sampling scores a window instead of the whole keyspace" do
entries = stringEntries(~w(k1 k2 k3 k4), "payload")
metadata = %{
"k1" => %{lastAccess: @now - 1_000, counter: 9},
"k2" => %{lastAccess: @now - 2_000, counter: 9},
"k3" => %{lastAccess: @now - 3_000, counter: 9},
"k4" => %{lastAccess: @now - 900_000, counter: 9}
}
opts = [now: @now, metadata: metadata, sampler: firstSampler()]
assert Eviction.selectVictim(entries, [policy: :allkeysLru, sampleSize: 1] ++ opts) ==
{:ok, "k1"}
assert Eviction.selectVictim(entries, [policy: :allkeysLru, sampleSize: 4] ++ opts) ==
{:ok, "k4"}
end
test "volatile-ttl evicts in expiry order and ignores keys without a TTL" do
entries = [
{"far", :string, "payload", @now + 900_000},
{"immortal", :string, "payload", nil},
{"near", :string, "payload", @now + 1_000},
{"mid", :string, "payload", @now + 60_000}
]
perEntry = Memory.entryBytes(hd(entries))
{table, keyspace} = fixtureKeyspace(entries)
assert {:ok, report} =
Eviction.run(
policy: :volatileTtl,
maxmemory: 2 * perEntry,
keyspace: keyspace,
now: @now,
sampleSize: 8
)
assert report.evicted == ["near", "mid"]
assert :ets.lookup(table, "immortal") != []
end
test "volatile candidates can run out before the limit is reached" do
entries =
stringEntries(~w(a1 a2 a3), "payload") ++
[
{"t1", :string, "payload", @now + 900_000},
{"t2", :string, "payload", @now + 900_000}
]
{table, keyspace} = fixtureKeyspace(entries)
assert {:error, report} =
Eviction.run(
policy: :volatileRandom,
maxmemory: 1,
keyspace: keyspace,
now: @now,
sampleSize: 8
)
assert report.reason == :exhausted
assert Enum.sort(report.evicted) == ["t1", "t2"]
assert report.usedMemory > 1
assert :ets.info(table, :size) == 3
end
test "allkeys-lfu backed by a live tracker keeps the hot key and forgets the victim" do
tracker = startTracker(logFactor: 0, decayTime: 0)
Enum.each(1..50, fn _index -> Tracker.touch(tracker, "hot", @now) end)
Tracker.touch(tracker, "cold", @now)
entries = stringEntries(["cold", "hot"], "payload")
coldBytes = Memory.entryBytes(hd(entries))
{table, keyspace} = fixtureKeyspace(entries)
assert {:ok, report} =
Eviction.run(
policy: :allkeysLfu,
maxmemory: Memory.usedMemory(entries) - coldBytes,
keyspace: keyspace,
tracker: tracker,
now: @now,
sampleSize: 8
)
assert report.evicted == ["cold"]
assert :ets.lookup(table, "hot") != []
assert Tracker.lookup(tracker, "cold", @now) == nil
assert Tracker.counter(tracker, "hot", @now) == 54
end
test "volatile-lru backed by a live tracker evicts the idle TTL key" do
tracker = startTracker(logFactor: 0, decayTime: 0)
Tracker.touch(tracker, "stale", @now - 300_000)
Tracker.touch(tracker, "fresh", @now - 1_000)
Tracker.touch(tracker, "idlest", @now - 900_000)
entries = [
{"fresh", :string, "payload", @now + 900_000},
{"stale", :string, "payload", @now + 900_000},
{"idlest", :string, "payload", nil}
]
assert Eviction.selectVictim(entries,
policy: :volatileLru,
tracker: tracker,
now: @now,
sampleSize: 8
) == {:ok, "stale"}
assert Eviction.selectVictim(entries,
policy: :allkeysLru,
tracker: tracker,
now: @now,
sampleSize: 8
) == {:ok, "idlest"}
end
test "a run against the live store really removes the keys" do
for index <- 1..20 do
:ok = Store.set("evict:#{index}", String.duplicate("payload", 20), [])
end
used = Memory.usedMemory()
target = div(used, 2)
assert {:ok, report} =
Eviction.run(policy: :allkeysRandom, maxmemory: target, seed: {1, 2, 3})
assert report.reason == :evicted
assert report.evictedCount >= 10
assert Memory.usedMemory() <= target
assert Memory.usedMemory() == report.usedMemory
assert Store.dbsize() == 20 - report.evictedCount
for key <- report.evicted do
assert Store.get(key) == nil
end
end
test "candidates carry the estimated size of the entry they stand for" do
entries = [
{"small", :string, "v", nil},
{"large", :string, :binary.copy("z", 4096), @now + 1_000}
]
[small, large] = Eviction.candidates(entries, :allkeysLru)
assert small.bytes == Memory.entryBytes({"small", :string, "v", nil})
assert large.bytes > 4096
assert large.expiresAt == @now + 1_000
assert Eviction.candidates(entries, :volatileLru) |> Enum.map(& &1.key) == ["large"]
assert Eviction.candidates(entries, :noeviction) == []
end
defp mixedEntries do
[
{"random-all", :string, "payload", nil},
{"idle-all", :string, "payload", nil},
{"cold-all", :string, "payload", nil},
{"random-ttl", :string, "payload", @now + 900_000},
{"idle-ttl", :string, "payload", @now + 900_000},
{"cold-ttl", :string, "payload", @now + 900_000},
{"soon-ttl", :string, "payload", @now + 1_000},
{"filler", :string, "payload", nil}
]
end
defp mixedMetadata do
%{
"random-all" => %{lastAccess: @now - 1_000, counter: 200},
"idle-all" => %{lastAccess: @now - 100_000, counter: 200},
"cold-all" => %{lastAccess: @now - 1_000, counter: 1},
"random-ttl" => %{lastAccess: @now - 1_000, counter: 200},
"idle-ttl" => %{lastAccess: @now - 90_000, counter: 200},
"cold-ttl" => %{lastAccess: @now - 1_000, counter: 2},
"soon-ttl" => %{lastAccess: @now - 1_000, counter: 200},
"filler" => %{lastAccess: @now - 500, counter: 250}
}
end
defp stringEntries(keys, value), do: Enum.map(keys, fn key -> {key, :string, value, nil} end)
defp fixtureKeyspace(entries) do
table = :ets.new(:evictionFixture, [:ordered_set, :public])
Enum.each(entries, &:ets.insert(table, &1))
{table,
%{
entries: fn -> :ets.tab2list(table) end,
delete: fn key -> :ets.delete(table, key) end
}}
end
defp firstSampler, do: fn pool, count, randState -> {Enum.take(pool, count), randState} end
defp windowSampler do
{:ok, cursor} = Agent.start_link(fn -> 0 end)
fn pool, count, randState ->
offset = Agent.get_and_update(cursor, fn pass -> {pass, pass + 1} end)
{pool |> Enum.drop(offset) |> Enum.take(count), randState}
end
end
defp startTracker(opts) do
id = {Tracker, System.unique_integer([:positive])}
start_supervised!(Supervisor.child_spec({Tracker, opts}, id: id))
end
end
+169
View File
@@ -0,0 +1,169 @@
defmodule Excache.Eviction.TrackerTest do
use ExUnit.Case, async: true
alias Excache.Eviction.Tracker
@minute 60_000
test "a fresh key starts at the Redis init counter with zero idle time" do
tracker = startTracker(logFactor: 0)
assert Tracker.touch(tracker, "alpha", 1_000) == Tracker.lfuInitCounter()
assert Tracker.lookup(tracker, "alpha", 1_000) == %{lastAccess: 1_000, idleMs: 0, counter: 5}
assert Tracker.lookup(tracker, "ghost", 1_000) == nil
assert Tracker.counter(tracker, "ghost", 1_000) == 0
assert Tracker.idleMs(tracker, "ghost", 1_000) == nil
end
test "a linear counter separates a key read a hundred times from one read twice" do
tracker = startTracker(logFactor: 0)
now = 1_000_000
touchTimes(tracker, "hot", 100, now)
touchTimes(tracker, "cold", 2, now)
assert Tracker.counter(tracker, "hot", now) == 104
assert Tracker.counter(tracker, "cold", now) == 6
end
test "the log factor compresses a hot counter far below its hit count" do
tracker = startTracker(logFactor: 10)
now = 1_000_000
touchTimes(tracker, "hot", 100, now)
touchTimes(tracker, "cold", 2, now)
hot = Tracker.counter(tracker, "hot", now)
cold = Tracker.counter(tracker, "cold", now)
assert cold == 6
assert hot > cold
assert hot < 30
end
test "decay strips the advantage of a hot key that goes idle" do
tracker = startTracker(logFactor: 0, decayTime: 1)
now = 1_000_000
touchTimes(tracker, "hot", 100, now)
touchTimes(tracker, "cold", 2, now)
partial = now + 5 * @minute
assert Tracker.counter(tracker, "hot", partial) == 99
assert Tracker.counter(tracker, "cold", partial) == 1
faded = now + 200 * @minute
assert Tracker.counter(tracker, "hot", faded) == 0
assert Tracker.counter(tracker, "cold", faded) == 0
Tracker.touch(tracker, "cold", faded)
assert Tracker.counter(tracker, "cold", faded) == 1
assert Tracker.counter(tracker, "cold", faded) > Tracker.counter(tracker, "hot", faded)
end
test "a decay time of zero freezes the counters" do
tracker = startTracker(logFactor: 0, decayTime: 0)
now = 1_000_000
touchTimes(tracker, "hot", 100, now)
assert Tracker.counter(tracker, "hot", now + 5000 * @minute) == 104
assert Tracker.settings(tracker) == %{logFactor: 0, decayTime: 0}
end
test "the counter saturates at the eight bit ceiling" do
tracker = startTracker(logFactor: 0, decayTime: 0)
now = 1_000_000
touchTimes(tracker, "hammered", 400, now)
assert Tracker.counter(tracker, "hammered", now) == Tracker.lfuMaxCounter()
assert Tracker.touch(tracker, "hammered", now) == 255
end
test "idle time is measured against the last access" do
tracker = startTracker(logFactor: 0)
now = 1_000_000
Tracker.touch(tracker, "stale", now)
Tracker.touch(tracker, "fresh", now + 30_000)
assert Tracker.idleMs(tracker, "stale", now + 30_000) == 30_000
assert Tracker.idleMs(tracker, "fresh", now + 30_000) == 0
assert Tracker.lookup(tracker, "stale", now - 5_000).idleMs == 0
end
test "forget, retain and clear prune the metadata" do
tracker = startTracker(logFactor: 0)
now = 1_000_000
Tracker.touchMany(tracker, ["a", "b", "c", "d"], now)
assert Tracker.size(tracker) == 4
Tracker.forget(tracker, "a")
assert Tracker.size(tracker) == 3
assert Tracker.lookup(tracker, "a", now) == nil
Tracker.retain(tracker, ["b", "c"])
assert Tracker.size(tracker) == 2
assert Tracker.lookup(tracker, "d", now) == nil
Tracker.clear(tracker)
assert Tracker.size(tracker) == 0
assert Tracker.snapshot(tracker, now) == %{}
end
test "metadata answers only for the requested keys and decays to the given time" do
tracker = startTracker(logFactor: 0, decayTime: 1)
now = 1_000_000
touchTimes(tracker, "hot", 10, now)
Tracker.touch(tracker, "cold", now)
Tracker.touch(tracker, "other", now)
metadata = Tracker.metadata(tracker, ["hot", "cold", "absent"], now + 3 * @minute)
assert Map.keys(metadata) |> Enum.sort() == ["cold", "hot"]
assert metadata["hot"].counter == 11
assert metadata["cold"].counter == 2
assert metadata["hot"].idleMs == 3 * @minute
assert Tracker.snapshot(tracker, now) |> Map.keys() |> Enum.sort() == ["cold", "hot", "other"]
end
test "touchMany registers each key once and returns the new counters" do
tracker = startTracker(logFactor: 0, decayTime: 0)
now = 1_000_000
assert Tracker.touchMany(tracker, ["a", "b"], now) == %{"a" => 5, "b" => 5}
assert Tracker.touchMany(tracker, ["a", "a"], now) == %{"a" => 7}
assert Tracker.counter(tracker, "a", now) == 7
end
test "the same seed replays the same logarithmic counter" do
now = 1_000_000
counters =
for _run <- 1..2 do
tracker = startTracker(logFactor: 10, seed: {3, 5, 7})
touchTimes(tracker, "hot", 250, now)
Tracker.counter(tracker, "hot", now)
end
assert [first, second] = counters
assert first == second
other = startTracker(logFactor: 10, seed: {11, 13, 17})
touchTimes(other, "hot", 250, now)
assert Tracker.counter(other, "hot", now) > 5
end
defp startTracker(opts) do
id = {Tracker, System.unique_integer([:positive])}
start_supervised!(Supervisor.child_spec({Tracker, opts}, id: id))
end
defp touchTimes(tracker, key, count, now) do
Enum.each(1..count, fn _index -> Tracker.touch(tracker, key, now) end)
end
end
+626
View File
@@ -0,0 +1,626 @@
defmodule Excache.IntrospectionTest do
use Excache.Test.ServerCase, async: false
alias Excache.Aof
alias Excache.Blocking
alias Excache.Command
alias Excache.Command.Introspection
alias Excache.Command.Introspection.Catalog
alias Excache.Command.Transaction
alias Excache.Config
alias Excache.Eviction.Runner
alias Excache.Memory
setup do
Config.reset()
Runner.resetStats()
on_exit(&Config.reset/0)
:ok
end
describe "INFO" do
test "answers with every default section in the documented order", %{conn: conn} do
assert {text, _conn} = Client.command(conn, ["INFO"])
assert sections(text) == [
"Server",
"Clients",
"Memory",
"Persistence",
"Stats",
"Replication",
"Keyspace"
]
assert String.ends_with?(text, "\r\n\r\n")
end
test "the server section reports the running system, not placeholders", %{conn: conn} do
assert {text, _conn} = Client.command(conn, ["INFO", "server"])
assert sections(text) == ["Server"]
values = fields(text)
assert values["redis_mode"] == "standalone"
assert values["excache_version"] == to_string(Application.spec(:excache, :vsn))
assert values["run_id"] =~ ~r/^[0-9a-f]{40}$/
assert values["arch_bits"] == Integer.to_string(:erlang.system_info(:wordsize) * 8)
assert values["process_id"] == to_string(:os.getpid())
assert values["elixir_version"] == System.version()
assert values["otp_release"] == to_string(:erlang.system_info(:otp_release))
assert String.to_integer(values["uptime_in_seconds"]) >= 0
end
test "used_memory matches what the memory accountant reports", %{conn: conn} do
assert {{:simple, "OK"}, conn} =
Client.command(conn, ["SET", "solo", String.duplicate("payload", 40)])
assert {text, _conn} = Client.command(conn, ["INFO", "memory"])
values = fields(text)
assert sections(text) == ["Memory"]
assert String.to_integer(values["used_memory"]) == Memory.usedMemory()
assert values["maxmemory"] == "0"
assert values["maxmemory_policy"] == "noeviction"
assert values["used_memory_human"] =~ ~r/^[0-9.]+[BKMG]$/
dataset = String.to_integer(values["used_memory_dataset"])
overhead = String.to_integer(values["used_memory_overhead"])
assert dataset + overhead == String.to_integer(values["used_memory"])
end
test "the keyspace section counts live keys, volatile keys and the mean ttl", %{conn: conn} do
assert {text, conn} = Client.command(conn, ["INFO", "keyspace"])
refute String.contains?(text, "db0")
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", "one"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "other", "two"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "timed", "three"])
assert {1, conn} = Client.command(conn, ["EXPIRE", "timed", "120"])
assert {text, _conn} = Client.command(conn, ["INFO", "keyspace"])
assert %{"db0" => summary} = fields(text)
assert [keys, expires, avgTtl] =
Regex.run(~r/keys=(\d+),expires=(\d+),avg_ttl=(\d+)/, summary,
capture: :all_but_first
)
assert keys == "3"
assert expires == "1"
assert String.to_integer(avgTtl) > 118_000
assert String.to_integer(avgTtl) <= 120_000
end
test "the persistence section reads the real append only file", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "durable", "value"])
:ok = Aof.flush()
assert {text, _conn} = Client.command(conn, ["INFO", "persistence"])
values = fields(text)
assert values["aof_enabled"] == "1"
assert values["aof_fsync_policy"] == "everysec"
assert values["loading"] == "0"
size = String.to_integer(values["aof_current_size"])
assert size > 0
assert size == File.stat!(Aof.path()).size
end
test "the stats section counts connections and eviction refusals", %{conn: conn, port: port} do
assert {text, conn} = Client.command(conn, ["INFO", "stats"])
before = String.to_integer(fields(text)["total_connections_received"])
extra = Client.connect(port)
assert {{:simple, "PONG"}, _extra} = Client.command(extra, ["PING"])
assert {text, conn} = Client.command(conn, ["INFO", "stats"])
values = fields(text)
assert String.to_integer(values["total_connections_received"]) == before + 1
assert String.to_integer(values["total_net_input_bytes"]) > 0
assert values["evicted_keys"] == "0"
assert values["oom_rejections"] == "0"
assert {{:simple, "OK"}, conn} =
Client.command(conn, ["SET", "filler", String.duplicate("x", 128)])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "1"])
assert {{:error, "OOM" <> _}, conn} = Client.command(conn, ["SET", "toobig", "value"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "0"])
assert {text, _conn} = Client.command(conn, ["INFO", "stats"])
assert String.to_integer(fields(text)["oom_rejections"]) > 0
Client.close(extra)
end
test "the replication section tells the truth about being a lone master", %{conn: conn} do
assert {text, _conn} = Client.command(conn, ["INFO", "replication"])
values = fields(text)
assert values["role"] == "master"
assert values["connected_slaves"] == "0"
assert values["master_failover_state"] == "no-failover"
assert values["repl_backlog_active"] == "0"
end
test "INFO all and INFO everything add the cpu section", %{conn: conn} do
assert {plain, conn} = Client.command(conn, ["INFO"])
refute "Cpu" in sections(plain)
assert {all, conn} = Client.command(conn, ["INFO", "all"])
assert "Cpu" in sections(all)
assert {everything, conn} = Client.command(conn, ["INFO", "everything"])
assert sections(everything) == sections(all)
values = fields(all)
assert String.to_integer(values["schedulers"]) == :erlang.system_info(:schedulers)
assert values["used_cpu_total"] =~ ~r/^\d+\.\d\d$/
assert {defaults, _conn} = Client.command(conn, ["INFO", "default"])
assert sections(defaults) == sections(plain)
end
test "several sections can be asked for at once and unknown ones are dropped", %{conn: conn} do
assert {text, conn} = Client.command(conn, ["INFO", "replication", "clients"])
assert sections(text) == ["Replication", "Clients"]
assert {"", conn} = Client.command(conn, ["INFO", "nosuchsection"])
assert {text, _conn} = Client.command(conn, ["INFO", "clients", "nosuchsection"])
assert sections(text) == ["Clients"]
end
test "blocked_clients counts a connection parked in BLPOP", %{conn: conn, port: port} do
assert clientField(conn, "blocked_clients") == 0
waiter = blockingTask(port, ["BLPOP", "parked", "0"])
awaitUntil(fn -> Blocking.waitingCount("parked") == 1 end)
assert clientField(conn, "blocked_clients") == 1
assert {1, conn} = Client.command(conn, ["LPUSH", "parked", "wake"])
assert Task.await(waiter) == ["parked", "wake"]
awaitUntil(fn -> clientField(conn, "blocked_clients") == 0 end)
end
test "pubsub_clients and pubsub_channels follow real subscriptions", %{
conn: conn,
port: port
} do
assert clientField(conn, "pubsub_clients") == 0
listener = Client.connect(port)
assert {["subscribe", "news", 1], _listener} =
Client.command(listener, ["SUBSCRIBE", "news"])
awaitUntil(fn -> clientField(conn, "pubsub_clients") == 1 end)
assert {text, _conn} = Client.command(conn, ["INFO", "stats"])
assert fields(text)["pubsub_channels"] == "1"
assert fields(text)["pubsub_patterns"] == "0"
Client.close(listener)
end
end
describe "CLIENT" do
test "ID hands out a distinct rising number per connection", %{conn: conn, port: port} do
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
assert is_integer(mine) and mine > 0
other = Client.connect(port)
assert {theirs, _other} = Client.command(other, ["CLIENT", "ID"])
assert theirs > mine
assert {^mine, _conn} = Client.command(conn, ["CLIENT", "ID"])
Client.close(other)
end
test "SETNAME and GETNAME round trip and reject illegal names", %{conn: conn} do
assert {"", conn} = Client.command(conn, ["CLIENT", "GETNAME"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "SETNAME", "importer"])
assert {"importer", conn} = Client.command(conn, ["CLIENT", "GETNAME"])
assert {{:error, "ERR Client names cannot contain spaces" <> _}, conn} =
Client.command(conn, ["CLIENT", "SETNAME", "two words"])
assert {"importer", conn} = Client.command(conn, ["CLIENT", "GETNAME"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "SETNAME", ""])
assert {"", _conn} = Client.command(conn, ["CLIENT", "GETNAME"])
end
test "LIST shows one line per connection with the real address and name", %{
conn: conn,
port: port
} do
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
other = Client.connect(port)
assert {theirs, other} = Client.command(other, ["CLIENT", "ID"])
assert {{:simple, "OK"}, _other} = Client.command(other, ["CLIENT", "SETNAME", "worker"])
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST"])
lines = listLines(text)
assert %{"name" => "worker", "addr" => address} = lines[theirs]
assert address =~ ~r/^127\.0\.0\.1:\d+$/
assert lines[theirs]["laddr"] =~ ~r/^127\.0\.0\.1:#{port}$/
assert lines[mine]["name"] == ""
assert lines[mine]["db"] == "0"
assert String.to_integer(lines[mine]["tot-net-in"]) > 0
assert {filtered, conn} =
Client.command(conn, ["CLIENT", "LIST", "ID", Integer.to_string(theirs)])
assert Map.keys(listLines(filtered)) == [theirs]
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["CLIENT", "LIST", "NOPE", "1"])
Client.close(other)
end
test "LIST TYPE separates subscribers from plain connections", %{conn: conn, port: port} do
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
listener = Client.connect(port)
assert {theirs, listener} = Client.command(listener, ["CLIENT", "ID"])
assert {["subscribe", "feed", 1], _listener} =
Client.command(listener, ["SUBSCRIBE", "feed"])
awaitUntil(fn ->
{text, _conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "pubsub"])
Map.has_key?(listLines(text), theirs)
end)
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "pubsub"])
assert Map.keys(listLines(text)) == [theirs]
assert listLines(text)[theirs]["sub"] == "1"
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "normal"])
assert Map.has_key?(listLines(text), mine)
refute Map.has_key?(listLines(text), theirs)
assert {"", conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "replica"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["CLIENT", "LIST", "TYPE", "nonsense"])
Client.close(listener)
end
test "INFO describes the calling connection only", %{conn: conn} do
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "SETNAME", "solo"])
assert {text, _conn} = Client.command(conn, ["CLIENT", "INFO"])
assert [{id, fieldMap}] = Map.to_list(listLines(text <> "\n"))
assert id == mine
assert fieldMap["name"] == "solo"
end
test "idle grows while a connection stays quiet and age tracks the connect time", %{
conn: conn,
port: port
} do
quiet = Client.connect(port)
assert {quietId, _quiet} = Client.command(quiet, ["CLIENT", "ID"])
Process.sleep(1_200)
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
assert {text, _conn} = Client.command(conn, ["CLIENT", "LIST"])
lines = listLines(text)
assert String.to_integer(lines[quietId]["idle"]) >= 1
assert String.to_integer(lines[quietId]["age"]) >= 1
assert lines[mine]["idle"] == "0"
Client.close(quiet)
end
test "KILL by ID tears down another live connection", %{conn: conn, port: port} do
victim = Client.connect(port)
assert {victimId, victim} = Client.command(victim, ["CLIENT", "ID"])
assert {{:simple, "PONG"}, victim} = Client.command(victim, ["PING"])
assert {1, conn} =
Client.command(conn, ["CLIENT", "KILL", "ID", Integer.to_string(victimId)])
assert {:error, :closed} = :gen_tcp.recv(victim.socket, 0, 1_000)
assert {0, conn} =
Client.command(conn, ["CLIENT", "KILL", "ID", Integer.to_string(victimId)])
assert {text, _conn} = Client.command(conn, ["CLIENT", "LIST"])
refute Map.has_key?(listLines(text), victimId)
end
test "KILL by address accepts the legacy single argument form", %{conn: conn, port: port} do
victim = Client.connect(port)
assert {victimId, victim} = Client.command(victim, ["CLIENT", "ID"])
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST"])
address = listLines(text)[victimId]["addr"]
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "KILL", address])
assert {:error, :closed} = :gen_tcp.recv(victim.socket, 0, 1_000)
assert {{:error, "ERR No such client address in the client list"}, _conn} =
Client.command(conn, ["CLIENT", "KILL", address])
end
test "KILL skips the caller unless SKIPME no is given", %{conn: conn, port: port} do
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
bystander = Client.connect(port)
assert {_id, _bystander} = Client.command(bystander, ["CLIENT", "ID"])
assert {0, conn} = Client.command(conn, ["CLIENT", "KILL", "ID", Integer.to_string(mine)])
assert {{:simple, "PONG"}, conn} = Client.command(conn, ["PING"])
assert {1, conn} =
Client.command(conn, [
"CLIENT",
"KILL",
"ID",
Integer.to_string(mine),
"SKIPME",
"no"
])
assert {:error, :closed} = :gen_tcp.recv(conn.socket, 0, 1_000)
Client.close(bystander)
end
test "KILL rejects filters it does not understand", %{conn: conn} do
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["CLIENT", "KILL", "WHATEVER", "1"])
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["CLIENT", "KILL", "ID", "notanumber"])
assert {{:error, "ERR Unknown subcommand" <> _}, _conn} =
Client.command(conn, ["CLIENT", "NOSUCHTHING"])
end
test "HELP lists the supported subcommands", %{conn: conn} do
assert {lines, _conn} = Client.command(conn, ["CLIENT", "HELP"])
assert {:simple, "KILL <ip:port>"} in lines
assert {:simple, "ID"} in lines
end
test "CLIENT off a real connection reports that it needs one" do
assert {[{:error, "ERR CLIENT is only available on a server connection"}], _state} =
Introspection.dispatch("CLIENT", ["ID"], commandState())
end
end
describe "COMMAND" do
test "the catalog covers exactly the commands the server dispatches" do
known = MapSet.new(Command.known_commands())
assert MapSet.new(Catalog.tableNames()) == known
end
test "the full reply carries one six element entry per command" do
assert {[{:array, entries}], _state} = Introspection.dispatch("COMMAND", [], commandState())
assert length(entries) == length(Enum.uniq(Command.known_commands()))
Enum.each(entries, fn {:array, entry} ->
assert [{:bulk, name}, {:integer, arity}, {:array, flags}, first, last, step] = entry
assert name == String.downcase(name)
assert arity != 0
assert Enum.all?(flags, &match?({:simple, _flag}, &1))
assert match?({:integer, _value}, first)
assert match?({:integer, _value}, last)
assert match?({:integer, _value}, step)
end)
end
test "COUNT and LIST agree with the full reply" do
assert {[{:integer, count}], _state} =
Introspection.dispatch("COMMAND", ["COUNT"], commandState())
assert {[{:array, names}], _state} =
Introspection.dispatch("COMMAND", ["LIST"], commandState())
assert count == length(names)
assert count == length(Enum.uniq(Command.known_commands()))
assert {:bulk, "append"} in names
end
test "INFO gives the real arity and key positions per command" do
assert commandInfo("GET") == {"get", 2, ["readonly", "fast"], 1, 1, 1}
assert commandInfo("SET") == {"set", -3, ["write", "denyoom"], 1, 1, 1}
assert commandInfo("MSET") == {"mset", -3, ["write", "denyoom"], 1, -1, 2}
assert commandInfo("DEL") == {"del", -2, ["write"], 1, -1, 1}
assert commandInfo("BLPOP") == {"blpop", -3, ["write", "noscript", "blocking"], 1, -2, 1}
assert commandInfo("BITOP") == {"bitop", -4, ["write", "denyoom"], 2, -1, 1}
assert commandInfo("SMOVE") == {"smove", 4, ["write", "fast"], 1, 2, 1}
assert commandInfo("PING") == {"ping", -1, ["fast"], 0, 0, 0}
assert {"xread", -4, flags, 0, 0, 0} = commandInfo("XREAD")
assert "movablekeys" in flags
end
test "INFO is case insensitive and answers nil for what it does not have" do
assert {[{:array, [entry, nil?]}], _state} =
Introspection.dispatch("COMMAND", ["INFO", "get", "nosuchcommand"], commandState())
assert {:array, [{:bulk, "get"} | _rest]} = entry
assert nil? == {:array, nil}
end
test "the declared arity matches what the dispatcher actually accepts", %{conn: conn} do
assert {{:error, "ERR wrong number of arguments for 'get' command"}, conn} =
Client.command(conn, ["GET", "a", "b"])
assert {{:error, "ERR wrong number of arguments for 'smove' command"}, conn} =
Client.command(conn, ["SMOVE", "a", "b"])
assert {{:error, "ERR wrong number of arguments for 'linsert' command"}, _conn} =
Client.command(conn, ["LINSERT", "a", "BEFORE", "b"])
end
test "GETKEYS walks plain ranges" do
assert getkeys(["GET", "one"]) == ["one"]
assert getkeys(["MSET", "a", "1", "b", "2", "c", "3"]) == ["a", "b", "c"]
assert getkeys(["DEL", "a", "b", "c"]) == ["a", "b", "c"]
assert getkeys(["BLPOP", "left", "right", "0"]) == ["left", "right"]
assert getkeys(["BITOP", "AND", "dest", "src1", "src2"]) == ["dest", "src1", "src2"]
assert getkeys(["SMOVE", "from", "to", "member"]) == ["from", "to"]
assert getkeys(["EXPIRE", "solo", "10"]) == ["solo"]
end
test "GETKEYS resolves the commands whose keys move" do
assert getkeys(["XREAD", "COUNT", "2", "STREAMS", "s1", "s2", "0", "0"]) == ["s1", "s2"]
assert getkeys([
"XREADGROUP",
"GROUP",
"g",
"c",
"STREAMS",
"stream",
">"
]) == ["stream"]
assert getkeys(["SINTERCARD", "2", "a", "b", "LIMIT", "1"]) == ["a", "b"]
assert getkeys(["BLMPOP", "0", "2", "a", "b", "LEFT"]) == ["a", "b"]
assert getkeys(["OBJECT", "ENCODING", "thing"]) == ["thing"]
assert getkeys(["MEMORY", "USAGE", "thing"]) == ["thing"]
assert getkeys(["XINFO", "STREAM", "events"]) == ["events"]
assert getkeys(["XGROUP", "CREATE", "events", "group", "$"]) == ["events"]
end
test "GETKEYS refuses what it cannot answer" do
assert getkeys(["PING"]) == {:error, "ERR The command has no key arguments"}
assert getkeys(["MEMORY", "DOCTOR"]) == {:error, "ERR The command has no key arguments"}
assert getkeys(["NOSUCHCOMMAND", "a"]) == {:error, "ERR Invalid command specified"}
assert getkeys(["GET"]) ==
{:error, "ERR Invalid number of arguments specified for command"}
assert getkeys(["XREAD", "COUNT", "2", "s1", "0"]) ==
{:error, "ERR Invalid arguments specified for command"}
assert getkeys(["SINTERCARD", "3", "a", "b"]) ==
{:error, "ERR Invalid arguments specified for command"}
end
test "DOCS answers with the group and arity taken from the same table" do
assert {[{:array, entries}], _state} =
Introspection.dispatch(
"COMMAND",
["DOCS", "zadd", "nosuchcommand"],
commandState()
)
assert entries == [
{:bulk, "zadd"},
{:array, [{:bulk, "group"}, {:bulk, "zset"}, {:bulk, "arity"}, {:integer, -4}]}
]
assert {[{:array, everything}], _state} =
Introspection.dispatch("COMMAND", ["DOCS"], commandState())
assert length(everything) == 2 * length(Enum.uniq(Command.known_commands()))
end
test "an unknown subcommand is named in the error" do
assert {[{:error, message}], _state} =
Introspection.dispatch("COMMAND", ["NOPE"], commandState())
assert message =~ "Unknown subcommand or wrong number of arguments for 'NOPE'"
end
test "the extended dispatcher already routes COMMAND to this module" do
assert {[{:integer, count}], _state} =
Excache.Command.Extended.dispatch("COMMAND", ["COUNT"], commandState())
assert count == Catalog.count()
end
end
defp commandState,
do: Map.put(Transaction.initial_state(), :subscriptions, MapSet.new())
defp commandInfo(name) do
assert {[{:array, [{:array, entry}]}], _state} =
Introspection.dispatch("COMMAND", ["INFO", name], commandState())
[
{:bulk, lowered},
{:integer, arity},
{:array, flags},
{:integer, first},
{:integer, last},
{:integer, step}
] = entry
{lowered, arity, Enum.map(flags, fn {:simple, flag} -> flag end), first, last, step}
end
defp getkeys(argv) do
case Introspection.dispatch("COMMAND", ["GETKEYS" | argv], commandState()) do
{[{:array, keys}], _state} -> Enum.map(keys, fn {:bulk, key} -> key end)
{[error], _state} -> error
end
end
defp clientField(conn, name) do
{text, _conn} = Client.command(conn, ["INFO", "clients"])
String.to_integer(fields(text)[name])
end
defp sections(text) do
for "# " <> title <- String.split(text, "\r\n"), do: title
end
defp fields(text) do
text
|> String.split("\r\n")
|> Enum.reject(fn line -> line == "" or String.starts_with?(line, "#") end)
|> Map.new(fn line ->
[key, value] = String.split(line, ":", parts: 2)
{key, value}
end)
end
defp listLines(text) do
text
|> String.split("\n")
|> Enum.reject(&(&1 == ""))
|> Map.new(fn line ->
parsed =
Map.new(String.split(line, " "), fn pair ->
[key, value] = String.split(pair, "=", parts: 2)
{key, value}
end)
{String.to_integer(parsed["id"]), parsed}
end)
end
defp blockingTask(port, args) do
Task.async(fn ->
client = Client.connect(port)
{reply, _client} = Client.command(client, args)
Client.close(client)
reply
end)
end
defp awaitUntil(check, attempts \\ 200) do
cond do
check.() -> :ok
attempts == 0 -> flunk("condition never held")
true -> Process.sleep(10) && awaitUntil(check, attempts - 1)
end
end
end
+214
View File
@@ -0,0 +1,214 @@
defmodule Excache.KeyspaceScanTest do
use Excache.Test.ServerCase, async: false
test "SCAN pages through the whole keyspace and returns every key once", %{conn: conn} do
conn = seed(conn, "key", 1..120)
{keys, calls, conn} = drain(conn, fn cursor -> ["SCAN", cursor] end)
assert calls > 1
assert length(keys) == 120
assert Enum.sort(keys) == Enum.sort(Enum.map(1..120, &"key:#{&1}"))
{[cursor, batch], _conn} = Client.command(conn, ["SCAN", "0", "COUNT", "1000"])
assert cursor == "0"
assert length(batch) == 120
end
test "SCAN filters with MATCH and TYPE", %{conn: conn} do
conn = seed(conn, "user", 1..30)
conn = seed(conn, "session", 1..30)
assert {1, conn} = Client.command(conn, ["RPUSH", "user:list", "a"])
{users, _calls, conn} = drain(conn, fn cursor -> ["SCAN", cursor, "MATCH", "user:*"] end)
assert length(users) == 31
{strings, _calls, conn} =
drain(conn, fn cursor -> ["SCAN", cursor, "MATCH", "user:*", "TYPE", "string"] end)
assert length(strings) == 30
refute "user:list" in strings
{lists, _calls, conn} = drain(conn, fn cursor -> ["SCAN", cursor, "TYPE", "list"] end)
assert lists == ["user:list"]
{none, _calls, _conn} = drain(conn, fn cursor -> ["SCAN", cursor, "MATCH", "nothing:*"] end)
assert none == []
end
test "SCAN returns keys that live through the whole iteration", %{conn: conn} do
conn = seed(conn, "stable", 1..80)
{[cursor, first], conn} = Client.command(conn, ["SCAN", "0"])
refute cursor == "0"
conn = seed(conn, "noise", 1..60)
assert {1, conn} = Client.command(conn, ["DEL", "noise:1"])
assert {1, conn} = Client.command(conn, ["DEL", "noise:2"])
{seen, _calls, _conn} = drainFrom(conn, fn value -> ["SCAN", value] end, cursor, first)
stable = Enum.filter(seen, &String.starts_with?(&1, "stable:"))
assert length(stable) == 80
assert length(Enum.uniq(stable)) == 80
end
test "SCAN validates the cursor", %{conn: conn} do
assert {{:error, "ERR invalid cursor"}, conn} = Client.command(conn, ["SCAN", "abc"])
assert {{:error, "ERR invalid cursor"}, conn} = Client.command(conn, ["SCAN", "-1"])
{[cursor, keys], conn} = Client.command(conn, ["SCAN", "9999"])
assert cursor == "0"
assert keys == []
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["SCAN", "0", "COUNT", "0"])
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
Client.command(conn, ["SCAN", "0", "COUNT", "many"])
assert {{:error, "ERR syntax error"}, conn} = Client.command(conn, ["SCAN", "0", "NOVALUES"])
assert {{:error, "ERR syntax error"}, conn} = Client.command(conn, ["SCAN", "0", "MATCH"])
assert {{:error, "ERR wrong number of arguments for 'scan' command"}, _conn} =
Client.command(conn, ["SCAN"])
end
test "HSCAN returns field value pairs, MATCH and NOVALUES", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["HSET", "h", "f1", "v1", "f2", "v2", "other", "v3"])
{[cursor, flat], conn} = Client.command(conn, ["HSCAN", "h", "0"])
assert cursor == "0"
assert flat |> Enum.chunk_every(2) |> Enum.sort() == [
["f1", "v1"],
["f2", "v2"],
["other", "v3"]
]
{[_cursor, fields], conn} = Client.command(conn, ["HSCAN", "h", "0", "NOVALUES"])
assert Enum.sort(fields) == ["f1", "f2", "other"]
{[_cursor, matched], conn} = Client.command(conn, ["HSCAN", "h", "0", "MATCH", "f*"])
assert matched |> Enum.chunk_every(2) |> Enum.sort() == [["f1", "v1"], ["f2", "v2"]]
{[_cursor, names], conn} =
Client.command(conn, ["HSCAN", "h", "0", "MATCH", "f?", "NOVALUES"])
assert Enum.sort(names) == ["f1", "f2"]
{[emptyCursor, empty], conn} = Client.command(conn, ["HSCAN", "nohash", "0"])
assert emptyCursor == "0"
assert empty == []
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "text", "v"])
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["HSCAN", "text", "0"])
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["HSCAN", "h", "0", "TYPE", "hash"])
assert {{:error, "ERR invalid cursor"}, _conn} = Client.command(conn, ["HSCAN", "h", "later"])
end
test "HSCAN pages through a large hash exactly once per field", %{conn: conn} do
fields = Enum.flat_map(1..300, fn index -> ["field:#{index}", "value:#{index}"] end)
assert {300, conn} = Client.command(conn, ["HSET", "big"] ++ fields)
{flat, calls, conn} = drain(conn, fn cursor -> ["HSCAN", "big", cursor] end)
pairs = Enum.chunk_every(flat, 2)
assert calls > 1
assert length(pairs) == 300
assert length(Enum.uniq(pairs)) == 300
assert ["field:7", "value:7"] in pairs
{sparse, sparseCalls, _conn} =
drain(conn, fn cursor -> ["HSCAN", "big", cursor, "COUNT", "500"] end)
assert sparseCalls == 1
assert length(sparse) == 600
end
test "SSCAN pages a set and keeps members that live through the iteration", %{conn: conn} do
members = Enum.map(1..200, &"member:#{&1}")
assert {200, conn} = Client.command(conn, ["SADD", "bag"] ++ members)
{[cursor, first], conn} = Client.command(conn, ["SSCAN", "bag", "0"])
refute cursor == "0"
assert {50, conn} = Client.command(conn, ["SADD", "bag"] ++ Enum.map(1..50, &"extra:#{&1}"))
assert {25, conn} = Client.command(conn, ["SREM", "bag"] ++ Enum.map(1..25, &"extra:#{&1}"))
{seen, _calls, conn} = drainFrom(conn, fn value -> ["SSCAN", "bag", value] end, cursor, first)
stable = Enum.filter(seen, &String.starts_with?(&1, "member:"))
assert length(stable) == 200
assert length(Enum.uniq(stable)) == 200
{[_cursor, matched], conn} =
Client.command(conn, ["SSCAN", "bag", "0", "MATCH", "member:1", "COUNT", "1000"])
assert matched == ["member:1"]
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["SSCAN", "bag", "0", "NOVALUES"])
end
test "ZSCAN returns members with their scores", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["ZADD", "board", "1", "a", "2.5", "b", "-3", "c"])
{[cursor, flat], conn} = Client.command(conn, ["ZSCAN", "board", "0"])
assert cursor == "0"
assert flat |> Enum.chunk_every(2) |> Enum.sort() == [["a", "1"], ["b", "2.5"], ["c", "-3"]]
{[_cursor, matched], conn} = Client.command(conn, ["ZSCAN", "board", "0", "MATCH", "a"])
assert matched == ["a", "1"]
{[emptyCursor, empty], conn} = Client.command(conn, ["ZSCAN", "nozset", "0"])
assert emptyCursor == "0"
assert empty == []
assert {1, conn} = Client.command(conn, ["RPUSH", "listkey", "x"])
assert {{:error, "WRONGTYPE" <> _}, _conn} = Client.command(conn, ["ZSCAN", "listkey", "0"])
end
test "ZSCAN pages a large sorted set exactly once per member", %{conn: conn} do
entries = Enum.flat_map(1..250, fn index -> [Integer.to_string(index), "member:#{index}"] end)
assert {250, conn} = Client.command(conn, ["ZADD", "ranking"] ++ entries)
{flat, calls, _conn} = drain(conn, fn cursor -> ["ZSCAN", "ranking", cursor] end)
pairs = Enum.chunk_every(flat, 2)
assert calls > 1
assert length(pairs) == 250
assert length(Enum.uniq(pairs)) == 250
assert ["member:42", "42"] in pairs
end
defp seed(conn, prefix, range) do
Enum.reduce(range, conn, fn index, current ->
{{:simple, "OK"}, next} = Client.command(current, ["SET", "#{prefix}:#{index}", "v"])
next
end)
end
defp drain(conn, buildArgs), do: drainFrom(conn, buildArgs, "0", [])
defp drainFrom(conn, buildArgs, cursor, acc), do: drainFrom(conn, buildArgs, cursor, acc, 0)
defp drainFrom(_conn, _buildArgs, _cursor, _acc, calls) when calls > 500,
do: flunk("scan never reached cursor 0")
defp drainFrom(conn, buildArgs, cursor, acc, calls) do
{[next, batch], conn} = Client.command(conn, buildArgs.(cursor))
collected = acc ++ batch
if next == "0" do
{collected, calls + 1, conn}
else
drainFrom(conn, buildArgs, next, collected, calls + 1)
end
end
end
+548
View File
@@ -0,0 +1,548 @@
defmodule Excache.KeyspaceTest do
use Excache.Test.ServerCase, async: false
test "STRLEN, GETRANGE and SETRANGE follow Redis offsets", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "greeting", "Hello World"])
assert {11, conn} = Client.command(conn, ["STRLEN", "greeting"])
assert {0, conn} = Client.command(conn, ["STRLEN", "absent"])
assert {"Hello", conn} = Client.command(conn, ["GETRANGE", "greeting", "0", "4"])
assert {"World", conn} = Client.command(conn, ["GETRANGE", "greeting", "-5", "-1"])
assert {"", conn} = Client.command(conn, ["GETRANGE", "greeting", "10", "4"])
assert {"", conn} = Client.command(conn, ["GETRANGE", "absent", "0", "-1"])
assert {11, conn} = Client.command(conn, ["SETRANGE", "greeting", "6", "Redis"])
assert {"Hello Redis", conn} = Client.command(conn, ["GETRANGE", "greeting", "0", "-1"])
assert {8, conn} = Client.command(conn, ["SETRANGE", "padded", "5", "abc"])
{padded, conn} = Client.command(conn, ["GET", "padded"])
assert padded == :binary.copy(<<0>>, 5) <> "abc"
assert {0, conn} = Client.command(conn, ["SETRANGE", "untouched", "0", ""])
assert {0, conn} = Client.command(conn, ["EXISTS", "untouched"])
assert {{:error, "ERR offset is out of range"}, _conn} =
Client.command(conn, ["SETRANGE", "greeting", "-1", "x"])
end
test "SETNX and GETDEL", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["SETNX", "slot", "first"])
assert {0, conn} = Client.command(conn, ["SETNX", "slot", "second"])
assert {"first", conn} = Client.command(conn, ["GET", "slot"])
assert {"first", conn} = Client.command(conn, ["GETDEL", "slot"])
assert {0, conn} = Client.command(conn, ["EXISTS", "slot"])
assert {nil, conn} = Client.command(conn, ["GETDEL", "slot"])
assert {1, conn} = Client.command(conn, ["RPUSH", "queue", "x"])
assert {{:error, "WRONGTYPE" <> _}, _conn} = Client.command(conn, ["GETDEL", "queue"])
end
test "INCRBYFLOAT accumulates and rejects non numeric values", %{conn: conn} do
assert {"10.5", conn} = Client.command(conn, ["INCRBYFLOAT", "balance", "10.5"])
assert {"10.6", conn} = Client.command(conn, ["INCRBYFLOAT", "balance", "0.1"])
assert {"8", conn} = Client.command(conn, ["INCRBYFLOAT", "balance", "-2.6"])
assert {"5", conn} = Client.command(conn, ["INCRBYFLOAT", "counter", "5"])
assert {{:error, "ERR value is not a valid float"}, conn} =
Client.command(conn, ["INCRBYFLOAT", "balance", "abc"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "word", "hello"])
assert {{:error, "ERR value is not a valid float"}, _conn} =
Client.command(conn, ["INCRBYFLOAT", "word", "1"])
end
test "HSETNX and HMGET", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["HSETNX", "profile", "name", "ann"])
assert {0, conn} = Client.command(conn, ["HSETNX", "profile", "name", "ben"])
assert {"ann", conn} = Client.command(conn, ["HGET", "profile", "name"])
assert {1, conn} = Client.command(conn, ["HSETNX", "profile", "city", "gdansk"])
{values, conn} = Client.command(conn, ["HMGET", "profile", "name", "ghost", "city"])
assert values == ["ann", nil, "gdansk"]
{absent, conn} = Client.command(conn, ["HMGET", "nohash", "a", "b"])
assert absent == [nil, nil]
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", "v"])
assert {{:error, "WRONGTYPE" <> _}, _conn} = Client.command(conn, ["HMGET", "plain", "a"])
end
test "SMISMEMBER and SINTERCARD", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["SADD", "left", "a", "b", "c"])
assert {3, conn} = Client.command(conn, ["SADD", "right", "b", "c", "d"])
{flags, conn} = Client.command(conn, ["SMISMEMBER", "left", "a", "zz", "c"])
assert flags == [1, 0, 1]
{none, conn} = Client.command(conn, ["SMISMEMBER", "noset", "a"])
assert none == [0]
assert {2, conn} = Client.command(conn, ["SINTERCARD", "2", "left", "right"])
assert {2, conn} = Client.command(conn, ["SINTERCARD", "2", "left", "right", "LIMIT", "0"])
assert {1, conn} = Client.command(conn, ["SINTERCARD", "2", "left", "right", "LIMIT", "1"])
assert {{:error, "ERR numkeys should be greater than 0"}, conn} =
Client.command(conn, ["SINTERCARD", "0", "left"])
assert {{:error, "ERR Number of keys can't be greater than number of args"}, conn} =
Client.command(conn, ["SINTERCARD", "3", "left"])
assert {{:error, "ERR LIMIT can't be negative"}, conn} =
Client.command(conn, ["SINTERCARD", "1", "left", "LIMIT", "-1"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["SINTERCARD", "1", "left", "BOUND", "1"])
end
test "SPOP replies with a bulk without count and an array with count", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["SADD", "bag", "a", "b", "c"])
{single, conn} = Client.command(conn, ["SPOP", "bag"])
assert single in ["a", "b", "c"]
{batch, conn} = Client.command(conn, ["SPOP", "bag", "5"])
assert length(batch) == 2
assert {0, conn} = Client.command(conn, ["EXISTS", "bag"])
assert {nil, conn} = Client.command(conn, ["SPOP", "bag"])
assert {[], conn} = Client.command(conn, ["SPOP", "bag", "3"])
assert {5, conn} = Client.command(conn, ["SADD", "bag", "a", "b", "c", "d", "e"])
assert {[], conn} = Client.command(conn, ["SPOP", "bag", "0"])
assert {5, conn} = Client.command(conn, ["SCARD", "bag"])
assert {{:error, "ERR value is out of range, must be positive"}, _conn} =
Client.command(conn, ["SPOP", "bag", "-1"])
end
test "SRANDMEMBER repeats members for a negative count", %{conn: conn} do
assert {2, conn} = Client.command(conn, ["SADD", "colors", "red", "green"])
{single, conn} = Client.command(conn, ["SRANDMEMBER", "colors"])
assert single in ["red", "green"]
{repeats, conn} = Client.command(conn, ["SRANDMEMBER", "colors", "-5"])
assert length(repeats) == 5
assert Enum.all?(repeats, &(&1 in ["red", "green"]))
{capped, conn} = Client.command(conn, ["SRANDMEMBER", "colors", "5"])
assert Enum.sort(capped) == ["green", "red"]
assert {2, conn} = Client.command(conn, ["SCARD", "colors"])
assert {nil, conn} = Client.command(conn, ["SRANDMEMBER", "nothing"])
assert {[], _conn} = Client.command(conn, ["SRANDMEMBER", "nothing", "3"])
end
test "SMOVE moves a member and refuses a destination of the wrong type", %{conn: conn} do
assert {2, conn} = Client.command(conn, ["SADD", "source", "x", "y"])
assert {1, conn} = Client.command(conn, ["SADD", "target", "z"])
assert {1, conn} = Client.command(conn, ["SMOVE", "source", "target", "x"])
assert {0, conn} = Client.command(conn, ["SMOVE", "source", "target", "ghost"])
{members, conn} = Client.command(conn, ["SMEMBERS", "target"])
assert Enum.sort(members) == ["x", "z"]
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "notaset", "v"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["SMOVE", "source", "notaset", "y"])
{left, _conn} = Client.command(conn, ["SMEMBERS", "source"])
assert left == ["y"]
end
test "LREM counts from the tail for a negative count", %{conn: conn} do
assert {5, conn} = Client.command(conn, ["RPUSH", "log", "a", "b", "a", "c", "a"])
assert {2, conn} = Client.command(conn, ["LREM", "log", "-2", "a"])
{rest, conn} = Client.command(conn, ["LRANGE", "log", "0", "-1"])
assert rest == ["a", "b", "c"]
assert {1, conn} = Client.command(conn, ["LREM", "log", "0", "a"])
assert {0, conn} = Client.command(conn, ["LREM", "log", "3", "zz"])
assert {0, conn} = Client.command(conn, ["LREM", "absent", "0", "a"])
assert {{:error, "ERR value is not an integer or out of range"}, _conn} =
Client.command(conn, ["LREM", "log", "many", "a"])
end
test "LSET reports missing keys and out of range indexes", %{conn: conn} do
assert {{:error, "ERR no such key"}, conn} =
Client.command(conn, ["LSET", "nolist", "0", "v"])
assert {3, conn} = Client.command(conn, ["RPUSH", "items", "a", "b", "c"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["LSET", "items", "-1", "z"])
{items, conn} = Client.command(conn, ["LRANGE", "items", "0", "-1"])
assert items == ["a", "b", "z"]
assert {{:error, "ERR index out of range"}, conn} =
Client.command(conn, ["LSET", "items", "9", "v"])
assert {{:error, "ERR index out of range"}, _conn} =
Client.command(conn, ["LSET", "items", "-9", "v"])
end
test "LTRIM keeps an inclusive window and drops an empty list", %{conn: conn} do
assert {5, conn} = Client.command(conn, ["RPUSH", "queue", "a", "b", "c", "d", "e"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["LTRIM", "queue", "1", "-2"])
{kept, conn} = Client.command(conn, ["LRANGE", "queue", "0", "-1"])
assert kept == ["b", "c", "d"]
assert {{:simple, "OK"}, conn} = Client.command(conn, ["LTRIM", "queue", "5", "10"])
assert {0, conn} = Client.command(conn, ["EXISTS", "queue"])
assert {{:simple, "OK"}, _conn} = Client.command(conn, ["LTRIM", "absent", "0", "-1"])
end
test "LINSERT places an element around a pivot", %{conn: conn} do
assert {2, conn} = Client.command(conn, ["RPUSH", "plan", "one", "three"])
assert {3, conn} = Client.command(conn, ["LINSERT", "plan", "BEFORE", "three", "two"])
assert {4, conn} = Client.command(conn, ["LINSERT", "plan", "AFTER", "three", "four"])
{plan, conn} = Client.command(conn, ["LRANGE", "plan", "0", "-1"])
assert plan == ["one", "two", "three", "four"]
assert {-1, conn} = Client.command(conn, ["LINSERT", "plan", "BEFORE", "ghost", "x"])
assert {0, conn} = Client.command(conn, ["LINSERT", "absent", "BEFORE", "a", "b"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["LINSERT", "plan", "SIDEWAYS", "one", "x"])
end
test "RPOPLPUSH moves, rotates and protects the destination", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["RPUSH", "source", "a", "b", "c"])
assert {"c", conn} = Client.command(conn, ["RPOPLPUSH", "source", "target"])
{target, conn} = Client.command(conn, ["LRANGE", "target", "0", "-1"])
assert target == ["c"]
assert {"b", conn} = Client.command(conn, ["RPOPLPUSH", "source", "source"])
{rotated, conn} = Client.command(conn, ["LRANGE", "source", "0", "-1"])
assert rotated == ["b", "a"]
assert {nil, conn} = Client.command(conn, ["RPOPLPUSH", "absent", "target"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "text", "v"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["RPOPLPUSH", "source", "text"])
{intact, _conn} = Client.command(conn, ["LRANGE", "source", "0", "-1"])
assert intact == ["b", "a"]
end
test "LPOS honours RANK, COUNT and MAXLEN", %{conn: conn} do
assert {8, conn} =
Client.command(conn, ["RPUSH", "mylist", "a", "b", "c", "1", "2", "3", "c", "c"])
assert {2, conn} = Client.command(conn, ["LPOS", "mylist", "c"])
assert {6, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "2"])
assert {7, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "-1"])
assert {6, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "-2"])
{all, conn} = Client.command(conn, ["LPOS", "mylist", "c", "COUNT", "0"])
assert all == [2, 6, 7]
{two, conn} = Client.command(conn, ["LPOS", "mylist", "c", "COUNT", "2"])
assert two == [2, 6]
{backwards, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "-1", "COUNT", "0"])
assert backwards == [7, 6, 2]
assert {nil, conn} = Client.command(conn, ["LPOS", "mylist", "ghost"])
{empty, conn} = Client.command(conn, ["LPOS", "mylist", "ghost", "COUNT", "0"])
assert empty == []
assert {nil, conn} = Client.command(conn, ["LPOS", "mylist", "c", "MAXLEN", "2"])
assert {2, conn} = Client.command(conn, ["LPOS", "mylist", "c", "MAXLEN", "3"])
assert {nil, conn} = Client.command(conn, ["LPOS", "absent", "c"])
assert {{:error, "ERR RANK can't be zero." <> _}, conn} =
Client.command(conn, ["LPOS", "mylist", "c", "RANK", "0"])
assert {{:error, "ERR COUNT can't be negative"}, conn} =
Client.command(conn, ["LPOS", "mylist", "c", "COUNT", "-1"])
assert {{:error, "ERR MAXLEN can't be negative"}, conn} =
Client.command(conn, ["LPOS", "mylist", "c", "MAXLEN", "-1"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["LPOS", "mylist", "c", "NEAR", "1"])
end
test "ZCOUNT handles exclusive and infinite bounds", %{conn: conn} do
assert {4, conn} =
Client.command(conn, ["ZADD", "prices", "10", "a", "20", "b", "30", "c", "40", "d"])
assert {4, conn} = Client.command(conn, ["ZCOUNT", "prices", "-inf", "+inf"])
assert {3, conn} = Client.command(conn, ["ZCOUNT", "prices", "20", "40"])
assert {2, conn} = Client.command(conn, ["ZCOUNT", "prices", "(20", "40"])
assert {1, conn} = Client.command(conn, ["ZCOUNT", "prices", "(20", "(40"])
assert {2, conn} = Client.command(conn, ["ZCOUNT", "prices", "-inf", "20"])
assert {0, conn} = Client.command(conn, ["ZCOUNT", "absent", "-inf", "+inf"])
assert {{:error, "ERR min or max is not a float"}, _conn} =
Client.command(conn, ["ZCOUNT", "prices", "abc", "10"])
end
test "ZREVRANGE, ZREVRANK and ZMSCORE", %{conn: conn} do
assert {3, conn} = Client.command(conn, ["ZADD", "board", "1", "a", "2", "b", "3.5", "c"])
{desc, conn} = Client.command(conn, ["ZREVRANGE", "board", "0", "-1"])
assert desc == ["c", "b", "a"]
{top, conn} = Client.command(conn, ["ZREVRANGE", "board", "0", "0"])
assert top == ["c"]
{tail, conn} = Client.command(conn, ["ZREVRANGE", "board", "-2", "-1"])
assert tail == ["b", "a"]
{scored, conn} = Client.command(conn, ["ZREVRANGE", "board", "0", "-1", "WITHSCORES"])
assert scored == ["c", "3.5", "b", "2", "a", "1"]
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["ZREVRANGE", "board", "0", "-1", "WITHVALUES"])
assert {0, conn} = Client.command(conn, ["ZREVRANK", "board", "c"])
assert {2, conn} = Client.command(conn, ["ZREVRANK", "board", "a"])
assert {nil, conn} = Client.command(conn, ["ZREVRANK", "board", "ghost"])
{scores, conn} = Client.command(conn, ["ZMSCORE", "board", "a", "ghost", "c"])
assert scores == ["1", nil, "3.5"]
{absent, _conn} = Client.command(conn, ["ZMSCORE", "nozset", "a"])
assert absent == [nil]
end
test "RENAME and RENAMENX", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "old", "payload"])
assert {1, conn} = Client.command(conn, ["EXPIRE", "old", "100"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["RENAME", "old", "fresh"])
assert {"payload", conn} = Client.command(conn, ["GET", "fresh"])
assert {0, conn} = Client.command(conn, ["EXISTS", "old"])
{ttl, conn} = Client.command(conn, ["TTL", "fresh"])
assert ttl > 0
assert {{:error, "ERR no such key"}, conn} = Client.command(conn, ["RENAME", "old", "other"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "taken", "t"])
assert {0, conn} = Client.command(conn, ["RENAMENX", "fresh", "taken"])
assert {"payload", conn} = Client.command(conn, ["GET", "fresh"])
assert {1, conn} = Client.command(conn, ["RENAMENX", "fresh", "free"])
assert {"payload", conn} = Client.command(conn, ["GET", "free"])
assert {{:error, "ERR no such key"}, _conn} = Client.command(conn, ["RENAMENX", "ghost", "x"])
end
test "COPY with REPLACE and DB", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "origin", "payload"])
assert {1, conn} = Client.command(conn, ["COPY", "origin", "clone"])
assert {"payload", conn} = Client.command(conn, ["GET", "clone"])
assert {0, conn} = Client.command(conn, ["COPY", "origin", "clone"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "origin", "changed"])
assert {1, conn} = Client.command(conn, ["COPY", "origin", "clone", "REPLACE"])
assert {"changed", conn} = Client.command(conn, ["GET", "clone"])
assert {1, conn} = Client.command(conn, ["COPY", "origin", "sameDb", "DB", "0"])
assert {"changed", conn} = Client.command(conn, ["GET", "sameDb"])
assert {{:error, "ERR DB index is out of range"}, conn} =
Client.command(conn, ["COPY", "origin", "elsewhere", "DB", "3"])
assert {{:error, "ERR source and destination objects are the same"}, conn} =
Client.command(conn, ["COPY", "origin", "origin"])
assert {0, conn} = Client.command(conn, ["COPY", "ghost", "anywhere"])
assert {{:error, "ERR syntax error"}, _conn} =
Client.command(conn, ["COPY", "origin", "clone", "FORCE"])
end
test "TOUCH and UNLINK count existing keys", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MSET", "t1", "1", "t2", "2"])
assert {2, conn} = Client.command(conn, ["TOUCH", "t1", "t2", "t3"])
assert {2, conn} = Client.command(conn, ["UNLINK", "t1", "t2", "t3"])
assert {0, conn} = Client.command(conn, ["EXISTS", "t1"])
assert {0, _conn} = Client.command(conn, ["TOUCH", "t1"])
end
test "EXPIREAT, EXPIRETIME and PEXPIRETIME", %{conn: conn} do
future = System.system_time(:second) + 100
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "session", "v"])
assert {-1, conn} = Client.command(conn, ["EXPIRETIME", "session"])
assert {-1, conn} = Client.command(conn, ["PEXPIRETIME", "session"])
assert {-2, conn} = Client.command(conn, ["EXPIRETIME", "ghost"])
assert {-2, conn} = Client.command(conn, ["PEXPIRETIME", "ghost"])
assert {0, conn} = Client.command(conn, ["EXPIREAT", "ghost", Integer.to_string(future)])
assert {1, conn} = Client.command(conn, ["EXPIREAT", "session", Integer.to_string(future)])
assert {^future, conn} = Client.command(conn, ["EXPIRETIME", "session"])
{millis, conn} = Client.command(conn, ["PEXPIRETIME", "session"])
assert millis == future * 1000
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "doomed", "v"])
assert {1, conn} = Client.command(conn, ["EXPIREAT", "doomed", "1000"])
assert {0, conn} = Client.command(conn, ["EXISTS", "doomed"])
assert {{:error, "ERR value is not an integer or out of range"}, _conn} =
Client.command(conn, ["EXPIREAT", "session", "soon"])
end
test "EXPIREAT respects NX, XX, GT and LT", %{conn: conn} do
base = System.system_time(:second) + 100
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "lease", "v"])
assert {0, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "XX"])
assert {0, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "GT"])
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "NX"])
assert {^base, conn} = Client.command(conn, ["EXPIRETIME", "lease"])
assert {0, conn} =
Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base + 50), "NX"])
later = base + 50
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(later), "GT"])
assert {^later, conn} = Client.command(conn, ["EXPIRETIME", "lease"])
assert {0, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "GT"])
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "LT"])
assert {^base, conn} = Client.command(conn, ["EXPIRETIME", "lease"])
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(later), "XX"])
assert {{:error, "ERR NX and XX, GT or LT options at the same time are not compatible"}, conn} =
Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "NX", "XX"])
assert {{:error, "ERR Unsupported option SOON"}, _conn} =
Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "soon"])
end
test "OBJECT reports encoding, refcount, idletime and help", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "number", "12345"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "shared", "42"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "padded", "007"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "short", "hello"])
assert {{:simple, "OK"}, conn} =
Client.command(conn, ["SET", "long", String.duplicate("x", 60)])
assert {"int", conn} = Client.command(conn, ["OBJECT", "ENCODING", "number"])
assert {"embstr", conn} = Client.command(conn, ["OBJECT", "ENCODING", "padded"])
assert {"embstr", conn} = Client.command(conn, ["OBJECT", "ENCODING", "short"])
assert {"raw", conn} = Client.command(conn, ["OBJECT", "ENCODING", "long"])
assert {1, conn} = Client.command(conn, ["RPUSH", "smallList", "a"])
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "smallList"])
bigList = ["RPUSH", "bigList"] ++ Enum.map(1..200, &Integer.to_string/1)
assert {200, conn} = Client.command(conn, bigList)
assert {"quicklist", conn} = Client.command(conn, ["OBJECT", "ENCODING", "bigList"])
assert {2, conn} = Client.command(conn, ["SADD", "ints", "1", "2"])
assert {"intset", conn} = Client.command(conn, ["OBJECT", "ENCODING", "ints"])
assert {2, conn} = Client.command(conn, ["SADD", "words", "alpha", "beta"])
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "words"])
assert {1, conn} = Client.command(conn, ["HSET", "hash", "field", "value"])
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "hash"])
assert {1, conn} = Client.command(conn, ["ZADD", "zset", "1", "a"])
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "zset"])
assert {1, conn} = Client.command(conn, ["OBJECT", "REFCOUNT", "short"])
assert {2_147_483_647, conn} = Client.command(conn, ["OBJECT", "REFCOUNT", "shared"])
assert {0, conn} = Client.command(conn, ["OBJECT", "IDLETIME", "short"])
assert {{:error, "ERR no such key"}, conn} =
Client.command(conn, ["OBJECT", "ENCODING", "ghost"])
assert {{:error, "ERR no such key"}, conn} =
Client.command(conn, ["OBJECT", "REFCOUNT", "ghost"])
{help, conn} = Client.command(conn, ["OBJECT", "HELP"])
assert Enum.any?(help, &match?({:simple, "ENCODING <key>"}, &1))
assert {{:error, "ERR Unknown subcommand" <> _}, _conn} =
Client.command(conn, ["OBJECT", "SIZE", "short"])
end
test "wrong arity is reported per command", %{conn: conn} do
assert {{:error, "ERR wrong number of arguments for 'strlen' command"}, conn} =
Client.command(conn, ["STRLEN"])
assert {{:error, "ERR wrong number of arguments for 'lset' command"}, conn} =
Client.command(conn, ["LSET", "key"])
assert {{:error, "ERR wrong number of arguments for 'hmget' command"}, conn} =
Client.command(conn, ["HMGET", "key"])
assert {{:error, "ERR wrong number of arguments for 'copy' command"}, conn} =
Client.command(conn, ["COPY", "key"])
assert {{:error, "ERR wrong number of arguments for 'unlink' command"}, conn} =
Client.command(conn, ["UNLINK"])
assert {{:error, "ERR wrong number of arguments for 'expiretime' command"}, _conn} =
Client.command(conn, ["EXPIRETIME", "a", "b"])
end
test "wrong types and non integer arguments are rejected", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["RPUSH", "listkey", "x"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "stringkey", "x"])
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["STRLEN", "listkey"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["GETRANGE", "listkey", "0", "-1"])
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["LPOS", "stringkey", "x"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["ZCOUNT", "stringkey", "-inf", "+inf"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["ZREVRANGE", "stringkey", "0", "-1"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["ZMSCORE", "stringkey", "a"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["LTRIM", "stringkey", "0", "-1"])
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["SPOP", "stringkey"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["SMISMEMBER", "stringkey", "a"])
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
Client.command(conn, ["GETRANGE", "stringkey", "a", "1"])
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
Client.command(conn, ["SETRANGE", "stringkey", "here", "v"])
assert {{:error, "ERR value is not an integer or out of range"}, _conn} =
Client.command(conn, ["SRANDMEMBER", "stringkey", "few"])
end
test "keyspace commands are queued and replayed inside MULTI", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "counter", "1"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["STRLEN", "counter"])
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["COPY", "counter", "backup"])
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["TOUCH", "counter", "backup"])
{replies, conn} = Client.command(conn, ["EXEC"])
assert replies == [1, 1, 2]
assert {"1", _conn} = Client.command(conn, ["GET", "backup"])
end
end
+194
View File
@@ -0,0 +1,194 @@
defmodule Excache.MemoryTest do
use ExUnit.Case, async: false
alias Excache.Memory
alias Excache.SkipList
alias Excache.Store
alias Excache.Stream, as: CacheStream
setup do
Store.flushdb()
on_exit(&Store.flushdb/0)
:ok
end
test "entry overhead matches the real cost of an ETS insert" do
entries = for index <- 1..2000, do: {"probe:#{index}", :string, "value-#{index}", nil}
table = :ets.new(:memoryProbe, [:set, :public])
before = :ets.info(table, :memory)
Enum.each(entries, &:ets.insert(table, &1))
measured = (:ets.info(table, :memory) - before) * Memory.wordSize()
:ets.delete(table)
estimated = Memory.usedMemory(entries)
assert measured > 0
assert_in_delta estimated / measured, 1.0, 0.02
end
test "reference counted binaries are counted beyond the flat term size" do
value = :binary.copy("x", 4096)
flat = :erts_debug.size(value) * Memory.wordSize()
assert flat < 200
assert Memory.estimateValue(:string, value) == flat + 4096
nested = [:binary.copy("a", 1024), :binary.copy("b", 1024)]
assert Memory.estimateValue(:list, nested) >= 2048
heapBinary = :binary.copy("y", 64)
assert Memory.estimateValue(:string, heapBinary) ==
:erts_debug.size(heapBinary) * Memory.wordSize()
end
test "estimates grow with the element count of every aggregate type" do
pairs = [
{:list, listValue(2), listValue(40)},
{:hash, hashValue(2), hashValue(40)},
{:set, setValue(2), setValue(40)},
{:zset, zsetValue(2), zsetValue(40)},
{:stream, streamValue(2), streamValue(40)}
]
for {kind, small, large} <- pairs do
assert Memory.estimateValue(kind, small) > 0, "#{kind} small estimate is not positive"
assert Memory.estimateValue(kind, large) > Memory.estimateValue(kind, small),
"#{kind} estimate did not grow with the element count"
end
end
test "element counts and payload bytes are reported per type" do
assert Memory.elementCount(:string, "abc") == 1
assert Memory.payloadBytes(:string, "abc") == 3
assert Memory.elementCount(:list, ["a", "bb"]) == 2
assert Memory.payloadBytes(:list, ["a", "bb"]) == 3
hash = %{"f1" => "v1", "f2" => "vvv2"}
assert Memory.elementCount(:hash, hash) == 2
assert Memory.payloadBytes(:hash, hash) == 2 + 2 + 2 + 4
set = MapSet.new(["a", "bb", "ccc"])
assert Memory.elementCount(:set, set) == 3
assert Memory.payloadBytes(:set, set) == 6
zset = zsetValue(2)
assert Memory.elementCount(:zset, zset) == 2
assert Memory.payloadBytes(:zset, zset) == 2 * (byte_size("member1") + Memory.wordSize())
stream = streamValue(2)
assert Memory.elementCount(:stream, stream) == 2
assert Memory.payloadBytes(:stream, stream) ==
2 * (2 * Memory.wordSize() + byte_size("field") + byte_size("value1"))
end
test "usedMemory sums every entry and usageByType splits them by kind" do
entries = [
{"s", :string, "value", nil},
{"l", :list, listValue(3), nil},
{"h", :hash, hashValue(3), nil},
{"st", :set, setValue(3), nil},
{"z", :zset, zsetValue(3), nil},
{"x", :stream, streamValue(3), nil}
]
assert Memory.usedMemory(entries) ==
Enum.reduce(entries, 0, fn entry, total -> total + Memory.entryBytes(entry) end)
byType = Memory.usageByType(entries)
assert Map.keys(byType) |> Enum.sort() == Enum.sort(Memory.kinds())
assert Enum.all?(byType, fn {_kind, usage} -> usage.keys == 1 end)
assert byType.string.bytes == Memory.entryBytes({"s", :string, "value", nil})
assert Enum.sum(Enum.map(byType, fn {_kind, usage} -> usage.bytes end)) ==
Memory.usedMemory(entries)
withoutStreams = Enum.reject(entries, fn {_key, kind, _value, _ea} -> kind == :stream end)
assert Memory.usageByType(withoutStreams).stream == %{keys: 0, bytes: 0}
end
test "stats separate overhead from dataset and point at the largest key" do
entries = [
{"tiny", :string, "a", nil},
{"huge", :string, :binary.copy("z", 8192), 1_700_000_000_000},
{"mid", :list, listValue(10), nil}
]
stats = Memory.stats(entries)
assert stats.keys == 3
assert stats.volatileKeys == 1
assert stats.usedMemory == Memory.usedMemory(entries)
assert stats.datasetBytes == stats.usedMemory - stats.overheadBytes
assert stats.overheadBytes == 3 * Memory.entryOverhead() + overheadKeyBytes(entries)
assert stats.bytesPerKey == div(stats.usedMemory, 3)
assert {"huge", huge} = stats.largestKey
assert huge >= 8192
assert Memory.stats([]).largestKey == nil
assert Memory.stats([]).bytesPerKey == 0
end
test "describe reports the volatile flag and separates payload from overhead" do
value = :binary.copy("p", 512)
described = Memory.describe("session:1", {:string, value, 1_700_000_000_000})
assert described.kind == :string
assert described.volatile
assert described.payloadBytes == 512
assert described.bytes == described.keyBytes + described.valueBytes + Memory.entryOverhead()
assert described.overheadBytes == described.bytes - 512
assert Memory.describe("k", {:string, "v", nil}).volatile == false
end
test "usage reads a live key and reports nil for a missing one" do
:ok = Store.set("alpha", "value", [])
{:ok, 3} = Store.rpush("beta", ["a", "b", "c"])
assert Memory.usage("alpha") == Memory.estimateEntry("alpha", Store.watch_snapshot("alpha"))
assert Memory.usage("ghost") == nil
assert Memory.describeKey("ghost") == nil
assert Memory.describeKey("beta").kind == :list
assert Memory.describeKey("beta").elements == 3
assert Memory.usedMemory() == Memory.usage("alpha") + Memory.usage("beta")
assert Memory.usageByType().list.keys == 1
assert Memory.stats().keys == 2
end
test "keyBytes grows with the key length" do
assert Memory.keyBytes(String.duplicate("k", 200)) > Memory.keyBytes("k")
assert Memory.entryOverhead() == 10 * Memory.wordSize()
end
defp overheadKeyBytes(entries) do
Enum.reduce(entries, 0, fn {key, _kind, _value, _ea}, total ->
total + Memory.keyBytes(key)
end)
end
defp listValue(count), do: Enum.map(1..count, &"item#{&1}")
defp hashValue(count), do: Map.new(1..count, fn index -> {"field#{index}", "value#{index}"} end)
defp setValue(count), do: MapSet.new(1..count, &"member#{&1}")
defp zsetValue(count) do
Enum.reduce(1..count, SkipList.new(), fn index, zset ->
SkipList.insert(zset, "member#{index}", index * 1.0)
end)
end
defp streamValue(count) do
Enum.reduce(1..count, CacheStream.new(), fn index, stream ->
{:ok, updated, _id} =
CacheStream.add(stream, {:full, index, 0}, ["field", "value#{index}"], 0)
updated
end)
end
end
+214
View File
@@ -0,0 +1,214 @@
defmodule Excache.ServerCommandTest do
use Excache.Test.ServerCase, async: false
alias Excache.Config
alias Excache.Eviction.Runner
setup do
Config.reset()
Runner.resetStats()
on_exit(&Config.reset/0)
:ok
end
test "CONFIG GET answers with the live values and follows the pattern", %{conn: conn} do
assert {["maxmemory", "0"], conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
assert {values, conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory*"])
assert values == [
"maxmemory",
"0",
"maxmemory-policy",
"noeviction",
"maxmemory-samples",
"5"
]
assert {[], conn} = Client.command(conn, ["CONFIG", "GET", "nothing-like-this"])
assert {merged, conn} = Client.command(conn, ["CONFIG", "GET", "lfu-log-factor", "maxmemory"])
assert merged == ["lfu-log-factor", "10", "maxmemory", "0"]
assert {deduped, conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory", "maxmemory"])
assert deduped == ["maxmemory", "0"]
assert {{:error, "ERR wrong number of arguments for 'config|get' command"}, _conn} =
Client.command(conn, ["CONFIG", "GET"])
end
test "CONFIG SET stores what CONFIG GET reads back", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "1mb"])
assert {["maxmemory", "1048576"], conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
assert Config.maxmemory() == 1_048_576
assert {{:simple, "OK"}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory-policy", "allkeys-lru"])
assert {["maxmemory-policy", "allkeys-lru"], conn} =
Client.command(conn, ["CONFIG", "GET", "maxmemory-policy"])
assert Config.policy() == :allkeysLru
assert {{:simple, "OK"}, conn} =
Client.command(conn, [
"CONFIG",
"SET",
"maxmemory",
"0",
"maxmemory-samples",
"16"
])
assert {["maxmemory", "0", "maxmemory-samples", "16"], _conn} =
Client.command(conn, ["CONFIG", "GET", "maxmemory", "maxmemory-samples"])
end
test "CONFIG SET reports every way it can be wrong", %{conn: conn} do
assert {{:error, "ERR CONFIG SET failed - argument must be a memory value" <> _}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory", "plenty"])
assert {{:error, "ERR CONFIG SET failed - argument must be one of the following:" <> _}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory-policy", "lru"])
assert {{:error, "ERR Unknown option or number of arguments for CONFIG SET - 'nosuch'"}, conn} =
Client.command(conn, ["CONFIG", "SET", "nosuch", "1"])
assert {{:error, "ERR CONFIG SET failed - can't set immutable config 'port'"}, conn} =
Client.command(conn, ["CONFIG", "SET", "port", "6380"])
assert {{:error, "ERR wrong number of arguments for 'config|set' command"}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory"])
assert {{:error, "ERR wrong number of arguments for 'config|set' command"}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory", "1kb", "lfu-log-factor"])
assert {{:error, "ERR Unknown subcommand" <> _}, conn} =
Client.command(conn, ["CONFIG", "REWRITE"])
assert {["maxmemory", "0"], _conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
end
test "a rejected pair leaves the whole CONFIG SET unapplied", %{conn: conn} do
assert {{:error, _reason}, conn} =
Client.command(conn, [
"CONFIG",
"SET",
"maxmemory",
"2mb",
"maxmemory-policy",
"nope"
])
assert {["maxmemory", "0", "maxmemory-policy", "noeviction"], _conn} =
Client.command(conn, ["CONFIG", "GET", "maxmemory", "maxmemory-policy"])
end
test "MEMORY USAGE reports the real cost of a key", %{conn: conn} do
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "small", "x"])
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "large", String.duplicate("x", 4096)])
assert {small, conn} = Client.command(conn, ["MEMORY", "USAGE", "small"])
assert {large, conn} = Client.command(conn, ["MEMORY", "USAGE", "large"])
assert small > 0
assert large > small + 4000
assert large == Excache.Memory.usage("large")
assert {nil, conn} = Client.command(conn, ["MEMORY", "USAGE", "ghost"])
assert {^large, conn} = Client.command(conn, ["MEMORY", "USAGE", "large", "SAMPLES", "5"])
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
Client.command(conn, ["MEMORY", "USAGE", "large", "SAMPLES", "many"])
assert {{:error, "ERR syntax error"}, conn} =
Client.command(conn, ["MEMORY", "USAGE", "large", "COUNT", "5"])
assert {{:error, "ERR Unknown subcommand" <> _}, _conn} =
Client.command(conn, ["MEMORY", "USAGE"])
end
test "MEMORY STATS adds up to the same numbers the store reports", %{conn: conn} do
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", String.duplicate("a", 200)])
{2, conn} = Client.command(conn, ["RPUSH", "queue", "one", "two"])
{1, conn} = Client.command(conn, ["SADD", "tags", "red"])
{1, conn} = Client.command(conn, ["EXPIRE", "plain", "600"])
{stats, conn} = stats(conn)
assert stats["keys.count"] == 3
assert stats["keys.volatile"] == 1
assert stats["type.string.keys"] == 1
assert stats["type.list.keys"] == 1
assert stats["type.set.keys"] == 1
assert stats["type.hash.keys"] == 0
assert stats["maxmemory"] == 0
assert stats["maxmemory.policy"] == "noeviction"
assert stats["maxmemory.samples"] == 5
assert stats["evicted.keys"] == 0
assert stats["oom.rejections"] == 0
assert stats["largest.key"] == "plain"
assert stats["total.allocated"] ==
stats["type.string.bytes"] + stats["type.list.bytes"] + stats["type.set.bytes"]
assert stats["total.allocated"] == Excache.Memory.usedMemory()
assert stats["dataset.bytes"] + stats["overhead.total"] == stats["total.allocated"]
assert stats["keys.bytes-per-key"] == div(stats["total.allocated"], 3)
assert stats["largest.key.bytes"] == Excache.Memory.usage("plain")
{_, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "4mb"])
{updated, _conn} = stats(conn)
assert updated["maxmemory"] == 4_194_304
end
test "MEMORY DOCTOR describes the store it is actually looking at", %{conn: conn} do
assert {"Excache holds no keys, there is nothing to report.", conn} =
Client.command(conn, ["MEMORY", "DOCTOR"])
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "hog", String.duplicate("z", 20_000)])
conn =
Enum.reduce(1..5, conn, fn index, acc ->
{{:simple, "OK"}, acc} = Client.command(acc, ["SET", "tiny:#{index}", "v"])
acc
end)
{report, conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
assert report =~ "6 keys"
assert report =~ "no limit"
assert report =~ "maxmemory-policy noeviction"
assert report =~ "maxmemory is not set"
assert report =~ "key 'hog' holds"
assert report =~ "percent of everything stored"
{_, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory-policy", "volatile-ttl"])
{_, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "1mb"])
{volatile, conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
assert volatile =~ "1048576 byte limit"
assert volatile =~ "only evicts keys with a TTL and none of the 6 keys has one"
refute volatile =~ "maxmemory is not set"
{1, conn} = Client.command(conn, ["EXPIRE", "hog", "600"])
{withTtl, _conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
refute withTtl =~ "only evicts keys with a TTL"
end
test "MEMORY and CONFIG are known commands inside transactions", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
assert {{:simple, "QUEUED"}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory", "2kb"])
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
assert {[{:simple, "OK"}, ["maxmemory", "2048"]], _conn} = Client.command(conn, ["EXEC"])
assert Config.maxmemory() == 2048
end
defp stats(conn) do
{flat, conn} = Client.command(conn, ["MEMORY", "STATS"])
{flat |> Enum.chunk_every(2) |> Map.new(fn [name, value] -> {name, value} end), conn}
end
end
+273
View File
@@ -0,0 +1,273 @@
defmodule Excache.StoreConsistencyTest do
use Excache.Test.ServerCase, async: false
alias Excache.Aof
alias Excache.Config
alias Excache.Eviction.Tracker
alias Excache.Store
@ttlMs 200
@holdMs 400
setup do
Tracker.clear(Tracker)
on_exit(fn ->
Config.reset()
Tracker.clear(Tracker)
end)
:ok
end
describe "RENAME must not drop the source when the destination write is refused" do
test "an expired source is reported instead of being silently swallowed" do
source = "cons:rename:src"
dest = keyInOtherShard("cons:rename:dst-", source)
assert :ok = Store.set(source, "payload", px: @ttlMs)
assert {:error, :nokey} =
duringSuspendedShard(dest, fn -> Store.rename(source, dest) end)
assert Store.exists([dest]) == 0
assert Store.exists([source]) == 0
end
test "an existing destination is never reported as overwritten when it was not" do
source = "cons:rename:live-src"
dest = keyInOtherShard("cons:rename:live-dst-", source)
assert :ok = Store.set(dest, "stale", [])
assert :ok = Store.set(source, "fresh", px: @ttlMs)
assert {:error, :nokey} =
duringSuspendedShard(dest, fn -> Store.rename(source, dest) end)
assert {:ok, "stale"} = Store.get(dest)
end
test "a healthy rename still carries the value and the remaining ttl" do
source = "cons:rename:ok-src"
dest = keyInOtherShard("cons:rename:ok-dst-", source)
assert :ok = Store.set(source, "payload", px: 60_000)
assert {:ok, "OK"} = Store.rename(source, dest)
assert {:ok, "payload"} = Store.get(dest)
assert Store.exists([source]) == 0
remaining = Store.ttl(dest, :seconds)
assert remaining > 0 and remaining <= 60
end
end
describe "RENAMENX must not drop the source when the destination write is refused" do
test "an expired source is reported instead of counted as a rename" do
source = "cons:renamenx:src"
dest = keyInOtherShard("cons:renamenx:dst-", source)
assert :ok = Store.set(source, "payload", px: @ttlMs)
assert {:error, :nokey} =
duringSuspendedShard(dest, fn -> Store.renamenx(source, dest) end)
assert Store.exists([dest]) == 0
end
test "an occupied destination still refuses the rename and keeps both keys" do
source = "cons:renamenx:busy-src"
dest = keyInOtherShard("cons:renamenx:busy-dst-", source)
assert :ok = Store.set(source, "payload", [])
assert :ok = Store.set(dest, "taken", [])
assert {:ok, 0} = Store.renamenx(source, dest)
assert {:ok, "payload"} = Store.get(source)
assert {:ok, "taken"} = Store.get(dest)
end
end
describe "COPY must not destroy the destination when the write is refused" do
test "an expiring source leaves the existing destination intact" do
source = "cons:copy:src"
dest = keyInOtherShard("cons:copy:dst-", source)
assert :ok = Store.set(dest, "keep-me", [])
assert :ok = Store.set(source, "payload", px: @ttlMs)
assert {:ok, 0} =
duringSuspendedShard(dest, fn -> Store.copy(source, dest, true) end)
assert {:ok, "keep-me"} = Store.get(dest)
end
test "replacing a destination of a different type still overwrites it" do
source = "cons:copy:swap-src"
dest = keyInOtherShard("cons:copy:swap-dst-", source)
assert {:ok, 1} = Store.rpush(dest, ["old"])
assert :ok = Store.set(source, "fresh", [])
assert {:ok, 1} = Store.copy(source, dest, true)
assert {:ok, "fresh"} = Store.get(dest)
assert Store.type(dest) == "string"
end
test "copying onto an occupied destination without replace changes nothing" do
source = "cons:copy:keep-src"
dest = keyInOtherShard("cons:copy:keep-dst-", source)
assert :ok = Store.set(source, "fresh", [])
assert :ok = Store.set(dest, "untouched", [])
assert {:ok, 0} = Store.copy(source, dest, false)
assert {:ok, "untouched"} = Store.get(dest)
end
end
describe "SPOP must reach the log as the removal it actually performed" do
test "the replayed log rebuilds exactly the surviving members", %{conn: conn} do
key = "cons:pop:members"
members = Enum.map(1..20, &"member#{&1}")
assert {20, conn} = Client.command(conn, ["SADD", key | members])
{taken, conn} = Client.command(conn, ["SPOP", key, "5"])
assert length(taken) == 5
{single, conn} = Client.command(conn, ["SPOP", key])
assert single in members
{survivors, conn} = Client.command(conn, ["SMEMBERS", key])
assert length(survivors) == 14
log = flushedLog()
refute log =~ "SPOP"
assert log =~ "SREM"
conn = replayFromScratch(conn)
{replayed, _conn} = Client.command(conn, ["SMEMBERS", key])
assert Enum.sort(replayed) == Enum.sort(survivors)
end
test "draining the whole set replays as the key being gone", %{conn: conn} do
key = "cons:pop:drained"
assert {3, conn} = Client.command(conn, ["SADD", key, "a", "b", "c"])
{taken, conn} = Client.command(conn, ["SPOP", key, "3"])
assert Enum.sort(taken) == ["a", "b", "c"]
assert {0, conn} = Client.command(conn, ["EXISTS", key])
log = flushedLog()
refute log =~ "SPOP"
conn = replayFromScratch(conn)
assert {0, _conn} = Client.command(conn, ["EXISTS", key])
end
test "popping from a missing set replays as a no-op", %{conn: conn} do
assert {nil, conn} = Client.command(conn, ["SPOP", "cons:pop:ghost"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "cons:pop:marker", "v"])
flushedLog()
conn = replayFromScratch(conn)
assert {0, conn} = Client.command(conn, ["EXISTS", "cons:pop:ghost"])
assert {"v", _conn} = Client.command(conn, ["GET", "cons:pop:marker"])
end
end
describe "OBJECT IDLETIME must report the tracked idle time" do
test "a key left alone for five seconds reports five", %{conn: conn} do
assert :ok = Config.set("maxmemory-policy", "allkeys-lru")
key = "cons:idle:tracked"
assert :ok = Store.set(key, "v", [])
Tracker.touch(Tracker, key, System.system_time(:millisecond) - 5_000)
assert {5, _conn} = Client.command(conn, ["OBJECT", "IDLETIME", key])
end
test "a key without access metadata reports zero rather than an error", %{conn: conn} do
key = "cons:idle:untracked"
assert :ok = Store.set(key, "v", [])
assert {0, _conn} = Client.command(conn, ["OBJECT", "IDLETIME", key])
end
test "a missing key is still refused", %{conn: conn} do
assert {{:error, "ERR no such key"}, _conn} =
Client.command(conn, ["OBJECT", "IDLETIME", "cons:idle:ghost"])
end
end
defp duringSuspendedShard(shardKey, work) do
shard = shardPid(shardKey)
parent = self()
:sys.suspend(shard)
worker =
Task.async(fn ->
send(parent, :entered)
work.()
end)
queued =
try do
assert_receive :entered, 100
waiting = awaitQueuedCall(shard, worker.pid)
Process.sleep(@holdMs)
waiting
after
:sys.resume(shard)
end
assert queued, "the worker never reached the suspended shard, the expiry window was not open"
Task.await(worker)
end
defp awaitQueuedCall(shard, workerPid) do
Enum.any?(1..500, fn _attempt ->
if callPending?(shard, workerPid) do
true
else
Process.sleep(1)
false
end
end)
end
defp callPending?(shard, workerPid) do
{:messages, pending} = Process.info(shard, :messages)
Enum.any?(pending, &match?({:"$gen_call", {^workerPid, _ref}, _request}, &1))
end
defp flushedLog do
Aof.flush()
{:ok, log} = File.read(Application.get_env(:excache, :aof_path))
log
end
defp replayFromScratch(conn) do
Store.flushdb()
Aof.replay(Application.get_env(:excache, :aof_path))
conn
end
defp shardIndex(key), do: rem(:erlang.phash2(key), Store.shard_count())
defp shardPid(key), do: Process.whereis(Store.shard_name(shardIndex(key)))
defp keyInOtherShard(prefix, reference) do
Enum.find_value(1..1024, fn suffix ->
candidate = prefix <> Integer.to_string(suffix)
if shardIndex(candidate) != shardIndex(reference), do: candidate
end)
end
end
+347
View File
@@ -0,0 +1,347 @@
defmodule Excache.StoreOpsRegressionTest do
use Excache.Test.ServerCase, async: false
alias Excache.Store
describe "SMOVE must not drop the member when the destination has the wrong type" do
test "the member stays in the source and the caller is told about the type clash" do
source = "regr:smove:src"
dest = "regr:smove:dst-string"
assert {:ok, 1} = Store.sadd(source, ["alpha"])
assert :ok = Store.set(dest, "not-a-set", [])
assert {:error, :wrongtype} = Store.smove(source, dest, "alpha")
assert {:ok, ["alpha"]} = Store.smembers(source)
assert {:ok, "not-a-set"} = Store.get(dest)
end
test "the destination type is checked before the source is touched" do
source = "regr:smove:absent-src"
dest = "regr:smove:dst-list"
assert {:ok, 1} = Store.rpush(dest, ["element"])
assert {:error, :wrongtype} = Store.smove(source, dest, "alpha")
assert {:ok, ["element"]} = Store.lrange(dest, 0, -1)
end
test "a wrong type on the source is reported and the destination is left alone" do
source = "regr:smove:src-string"
dest = "regr:smove:dst-set"
assert :ok = Store.set(source, "not-a-set", [])
assert {:ok, 1} = Store.sadd(dest, ["beta"])
assert {:error, :wrongtype} = Store.smove(source, dest, "alpha")
assert {:ok, ["beta"]} = Store.smembers(dest)
end
test "a healthy move across two shards still transfers the member exactly once" do
source = "regr:smove:move-src"
dest = keyInOtherShard("regr:smove:move-dst-", source)
assert {:ok, 2} = Store.sadd(source, ["alpha", "gamma"])
assert {:ok, 1} = Store.sadd(dest, ["beta"])
assert {:ok, 1} = Store.smove(source, dest, "alpha")
assert {:ok, ["gamma"]} = Store.smembers(source)
assert {:ok, members} = Store.smembers(dest)
assert Enum.sort(members) == ["alpha", "beta"]
end
test "moving a member the source does not hold reports zero and adds nothing" do
source = "regr:smove:miss-src"
dest = "regr:smove:miss-dst"
assert {:ok, 1} = Store.sadd(source, ["alpha"])
assert {:ok, 1} = Store.sadd(dest, ["beta"])
assert {:ok, 0} = Store.smove(source, dest, "nothing-like-this")
assert {:ok, ["alpha"]} = Store.smembers(source)
assert {:ok, ["beta"]} = Store.smembers(dest)
end
test "SMOVE over RESP reports WRONGTYPE and keeps the member", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["SADD", "regr:smove:resp-src", "alpha"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "regr:smove:resp-dst", "s"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, [
"SMOVE",
"regr:smove:resp-src",
"regr:smove:resp-dst",
"alpha"
])
assert {["alpha"], _conn} = Client.command(conn, ["SMEMBERS", "regr:smove:resp-src"])
end
end
describe "RPOPLPUSH must not drop the element when the destination has the wrong type" do
test "the element stays at the tail of the source and the caller sees the error" do
source = "regr:rpoplpush:src"
dest = "regr:rpoplpush:dst-string"
assert {:ok, 2} = Store.rpush(source, ["head", "tail"])
assert :ok = Store.set(dest, "not-a-list", [])
assert {:error, :wrongtype} = Store.rpoplpush(source, dest)
assert {:ok, ["head", "tail"]} = Store.lrange(source, 0, -1)
assert {:ok, "not-a-list"} = Store.get(dest)
end
test "the destination type is checked before the source is popped" do
source = "regr:rpoplpush:absent-src"
dest = "regr:rpoplpush:dst-set"
assert {:ok, 1} = Store.sadd(dest, ["beta"])
assert {:error, :wrongtype} = Store.rpoplpush(source, dest)
assert {:ok, ["beta"]} = Store.smembers(dest)
end
test "a healthy rotation across two shards moves the tail to the head" do
source = "regr:rpoplpush:rot-src"
dest = keyInOtherShard("regr:rpoplpush:rot-dst-", source)
assert {:ok, 2} = Store.rpush(source, ["head", "tail"])
assert {:ok, 1} = Store.rpush(dest, ["existing"])
assert {:ok, "tail"} = Store.rpoplpush(source, dest)
assert {:ok, ["head"]} = Store.lrange(source, 0, -1)
assert {:ok, ["tail", "existing"]} = Store.lrange(dest, 0, -1)
end
test "an empty source yields nil and never creates the destination" do
source = "regr:rpoplpush:empty-src"
dest = "regr:rpoplpush:empty-dst"
assert {:ok, nil} = Store.rpoplpush(source, dest)
assert 0 == Store.exists([dest])
end
test "RPOPLPUSH over RESP reports WRONGTYPE and keeps the element", %{conn: conn} do
assert {1, conn} = Client.command(conn, ["RPUSH", "regr:rpoplpush:resp-src", "tail"])
assert {{:simple, "OK"}, conn} =
Client.command(conn, ["SET", "regr:rpoplpush:resp-dst", "s"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, [
"RPOPLPUSH",
"regr:rpoplpush:resp-src",
"regr:rpoplpush:resp-dst"
])
assert {["tail"], _conn} =
Client.command(conn, ["LRANGE", "regr:rpoplpush:resp-src", "0", "-1"])
end
end
describe "LPOS with a nil count must not crash the shard" do
test "a nil count means the first match and the shard survives the call" do
key = "regr:lpos:nil-count"
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
shardBefore = shardPid(key)
assert {:ok, 1} = Store.lpos(key, "b", 1, nil, 0)
assert shardPid(key) == shardBefore
assert Process.alive?(shardBefore)
assert {:ok, ["a", "b", "c", "b"]} = Store.lrange(key, 0, -1)
end
test "a nil count must not take the shard down and wipe its other keys" do
key = "regr:lpos:blast-radius"
neighbour = keyInSameShard("regr:lpos:neighbour-", key)
assert :ok = Store.set(neighbour, "survivor", [])
assert {:ok, 1} = Store.rpush(key, ["only"])
shardBefore = shardPid(key)
assert {:ok, 0} = Store.lpos(key, "only", 1, nil, 0)
assert shardPid(key) == shardBefore
assert {:ok, "survivor"} = Store.get(neighbour)
end
test "a nil count with no match yields nil rather than an empty list" do
key = "regr:lpos:no-match"
assert {:ok, 3} = Store.rpush(key, ["a", "b", "c"])
assert {:ok, nil} = Store.lpos(key, "zzz", 1, nil, 0)
assert {:ok, nil} = Store.lpos("regr:lpos:missing-key", "a", 1, nil, 0)
end
test "a nil count honours a positive rank" do
key = "regr:lpos:rank-positive"
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
assert {:ok, 1} = Store.lpos(key, "b", 1, nil, 0)
assert {:ok, 3} = Store.lpos(key, "b", 2, nil, 0)
assert {:ok, nil} = Store.lpos(key, "b", 3, nil, 0)
end
test "a nil count honours a negative rank and searches from the tail" do
key = "regr:lpos:rank-negative"
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
assert {:ok, 3} = Store.lpos(key, "b", -1, nil, 0)
assert {:ok, 1} = Store.lpos(key, "b", -2, nil, 0)
assert {:ok, nil} = Store.lpos(key, "b", -3, nil, 0)
end
test "a nil count honours maxlen" do
key = "regr:lpos:maxlen"
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
assert {:ok, 1} = Store.lpos(key, "b", 1, nil, 2)
assert {:ok, nil} = Store.lpos(key, "b", 1, nil, 1)
end
test "a zero count still means every match, in both rank directions" do
key = "regr:lpos:count-zero"
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
assert {:ok, [1, 3]} = Store.lpos(key, "b", 1, 0, 0)
assert {:ok, [3, 1]} = Store.lpos(key, "b", -1, 0, 0)
assert {:ok, [1]} = Store.lpos(key, "b", 1, 1, 0)
end
test "a nil count on a wrong type key reports WRONGTYPE instead of crashing" do
key = "regr:lpos:wrongtype"
assert :ok = Store.set(key, "not-a-list", [])
shardBefore = shardPid(key)
assert {:error, :wrongtype} = Store.lpos(key, "a", 1, nil, 0)
assert shardPid(key) == shardBefore
end
test "LPOS over RESP is unaffected by the fix", %{conn: conn} do
assert {4, conn} = Client.command(conn, ["RPUSH", "regr:lpos:resp", "a", "b", "c", "b"])
assert {1, conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "b"])
assert {nil, conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "zzz"])
assert {3, conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "b", "RANK", "-1"])
assert {[1, 3], _conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "b", "COUNT", "0"])
end
end
describe "HMGET and SMISMEMBER must surface WRONGTYPE" do
test "HMGET on a key of the wrong type reports the error instead of nils" do
key = "regr:hmget:wrongtype"
assert :ok = Store.set(key, "not-a-hash", [])
assert {:error, :wrongtype} = Store.hmget(key, ["field-a", "field-b"])
end
test "HMGET keeps returning a bare list for hashes and missing keys" do
key = "regr:hmget:hash"
assert {:ok, 1} = Store.hset(key, [{"field-a", "value-a"}])
assert ["value-a", nil] = Store.hmget(key, ["field-a", "field-b"])
assert [nil, nil] = Store.hmget("regr:hmget:missing", ["field-a", "field-b"])
end
test "SMISMEMBER on a key of the wrong type reports the error instead of zeros" do
key = "regr:smismember:wrongtype"
assert :ok = Store.set(key, "not-a-set", [])
assert {:error, :wrongtype} = Store.smismember(key, ["alpha", "beta"])
end
test "SMISMEMBER keeps returning a bare list for sets and missing keys" do
key = "regr:smismember:set"
assert {:ok, 1} = Store.sadd(key, ["alpha"])
assert [1, 0] = Store.smismember(key, ["alpha", "beta"])
assert [0, 0] = Store.smismember("regr:smismember:missing", ["alpha", "beta"])
end
test "HMGET and SMISMEMBER over RESP still report WRONGTYPE", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "regr:mixed:string", "v"])
assert {{:error, "WRONGTYPE" <> _}, conn} =
Client.command(conn, ["HMGET", "regr:mixed:string", "field-a"])
assert {{:error, "WRONGTYPE" <> _}, _conn} =
Client.command(conn, ["SMISMEMBER", "regr:mixed:string", "alpha"])
end
end
describe "SETRANGE with an empty value must not create a key" do
test "an empty value on a missing key returns zero and creates nothing" do
key = "regr:setrange:missing"
assert {:ok, 0} = Store.setrange(key, 5, "")
assert 0 == Store.exists([key])
assert "none" == Store.type(key)
end
test "an empty value on an existing string returns its length and changes nothing" do
key = "regr:setrange:existing"
assert :ok = Store.set(key, "hello", [])
assert {:ok, 5} = Store.setrange(key, 100, "")
assert {:ok, "hello"} = Store.get(key)
assert 1 == Store.exists([key])
end
test "an empty value on a key of the wrong type still reports WRONGTYPE" do
key = "regr:setrange:wrongtype"
assert {:ok, 1} = Store.rpush(key, ["element"])
assert {:error, :wrongtype} = Store.setrange(key, 0, "")
assert {:ok, ["element"]} = Store.lrange(key, 0, -1)
end
test "an empty value leaves an existing expiry in place" do
key = "regr:setrange:ttl"
assert :ok = Store.set(key, "hello", [])
assert 1 == Store.expire(key, 60_000)
assert {:ok, 5} = Store.setrange(key, 0, "")
assert Store.ttl(key, :seconds) > 0
end
test "a non empty value still pads and overwrites as before" do
key = "regr:setrange:write"
assert {:ok, 8} = Store.setrange(key, 5, "abc")
assert {:ok, <<0, 0, 0, 0, 0, "abc">>} = Store.get(key)
assert {:ok, 8} = Store.setrange(key, 0, "xy")
assert {:ok, <<"xy", 0, 0, 0, "abc">>} = Store.get(key)
end
test "SETRANGE with an empty value over RESP returns the length and creates nothing",
%{conn: conn} do
assert {0, conn} = Client.command(conn, ["SETRANGE", "regr:setrange:resp", "5", ""])
assert {0, conn} = Client.command(conn, ["EXISTS", "regr:setrange:resp"])
assert {{:simple, "OK"}, conn} =
Client.command(conn, ["SET", "regr:setrange:resp", "hello"])
assert {5, _conn} = Client.command(conn, ["SETRANGE", "regr:setrange:resp", "100", ""])
end
end
defp shardIndex(key), do: rem(:erlang.phash2(key), Store.shard_count())
defp shardPid(key), do: Process.whereis(Store.shard_name(shardIndex(key)))
defp keyInOtherShard(prefix, reference) do
findKey(prefix, fn candidate -> shardIndex(candidate) != shardIndex(reference) end)
end
defp keyInSameShard(prefix, reference) do
findKey(prefix, fn candidate -> shardIndex(candidate) == shardIndex(reference) end)
end
defp findKey(prefix, accept) do
Enum.find_value(1..1024, fn suffix ->
candidate = prefix <> Integer.to_string(suffix)
if accept.(candidate), do: candidate
end)
end
end
+265
View File
@@ -0,0 +1,265 @@
defmodule Excache.WritePathTest do
use Excache.Test.ServerCase, async: false
alias Excache.Aof
alias Excache.Blocking
alias Excache.Command
alias Excache.Config
alias Excache.Eviction.Runner
alias Excache.Eviction.Tracker
alias Excache.Store
@payload String.duplicate("x", 1000)
@oom {:error, "OOM command not allowed when used memory > 'maxmemory'"}
describe "renaming a key onto itself keeps it" do
test "the store answers OK and the value survives" do
assert :ok = Store.set("self:string", "payload", [])
assert {:ok, "OK"} = Store.rename("self:string", "self:string")
assert {:ok, "payload"} = Store.get("self:string")
end
test "a missing key renamed onto itself is still refused" do
assert {:error, :nokey} = Store.rename("self:ghost", "self:ghost")
end
test "the remaining ttl is untouched" do
assert :ok = Store.set("self:ttl", "payload", px: 60_000)
assert {:ok, "OK"} = Store.rename("self:ttl", "self:ttl")
remaining = Store.ttl("self:ttl", :seconds)
assert remaining > 0 and remaining <= 60
end
test "a list survives the round trip over the wire", %{conn: conn} do
assert {2, conn} = Client.command(conn, ["RPUSH", "self:list", "a", "b"])
assert {{:simple, "OK"}, conn} = Client.command(conn, ["RENAME", "self:list", "self:list"])
assert {["a", "b"], conn} = Client.command(conn, ["LRANGE", "self:list", "0", "-1"])
conn = replayFromScratch(conn)
assert {["a", "b"], _conn} = Client.command(conn, ["LRANGE", "self:list", "0", "-1"])
end
test "RENAMENX onto itself answers zero and keeps the key", %{conn: conn} do
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "self:nx", "v"])
assert {0, conn} = Client.command(conn, ["RENAMENX", "self:nx", "self:nx"])
assert {"v", _conn} = Client.command(conn, ["GET", "self:nx"])
end
end
describe "blocking moves obey the out of memory gate" do
setup do
Application.put_env(:excache, :eviction_recheck, 0)
Config.reset()
Tracker.clear(Tracker)
Runner.resetStats()
on_exit(fn ->
Config.reset()
Application.delete_env(:excache, :eviction_recheck)
end)
:ok
end
test "BRPOPLPUSH is refused once maxmemory is crossed", %{conn: conn} do
conn = fillOverLimit(conn, "oom:brpoplpush")
assert {@oom, conn} =
Client.command(conn, ["BRPOPLPUSH", "oom:brpoplpush", "oom:moved", "0"])
assert {1, conn} = Client.command(conn, ["LLEN", "oom:brpoplpush"])
assert {0, _conn} = Client.command(conn, ["EXISTS", "oom:moved"])
end
test "BLMOVE is refused once maxmemory is crossed", %{conn: conn} do
conn = fillOverLimit(conn, "oom:blmove")
assert {@oom, conn} =
Client.command(conn, ["BLMOVE", "oom:blmove", "oom:moved", "LEFT", "RIGHT", "0"])
assert {1, conn} = Client.command(conn, ["LLEN", "oom:blmove"])
assert {0, _conn} = Client.command(conn, ["EXISTS", "oom:moved"])
end
test "lifting the limit lets the same move through", %{conn: conn} do
conn = fillOverLimit(conn, "oom:lifted")
assert {@oom, conn} =
Client.command(conn, ["BLMOVE", "oom:lifted", "oom:dst", "LEFT", "RIGHT", "0"])
{{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "0"])
assert {"value", conn} =
Client.command(conn, ["BLMOVE", "oom:lifted", "oom:dst", "LEFT", "RIGHT", "0"])
assert {["value"], _conn} = Client.command(conn, ["LRANGE", "oom:dst", "0", "-1"])
end
end
describe "blocking writes reach the shared log funnel" do
test "a blocked BLMOVE announces the push it performed", %{conn: conn, port: port} do
subscriber = subscribeTo(port, "__keyevent@0__:rpush")
waiter = blockingTask(port, ["BLMOVE", "funnel:src", "funnel:dst", "LEFT", "RIGHT", "0"])
awaitWaiters("funnel:src", 1)
assert {1, _conn} = Client.command(conn, ["LPUSH", "funnel:src", "item"])
assert Task.await(waiter) == "item"
assert {["message", "__keyevent@0__:rpush", "funnel:dst"], _subscriber} =
Client.read(subscriber)
Client.close(subscriber)
end
test "an immediately served BLMOVE announces the push as well", %{conn: conn, port: port} do
assert {1, conn} = Client.command(conn, ["RPUSH", "inline:src", "item"])
subscriber = subscribeTo(port, "__keyevent@0__:rpush")
assert {"item", _conn} =
Client.command(conn, ["BLMOVE", "inline:src", "inline:dst", "LEFT", "RIGHT", "0"])
assert {["message", "__keyevent@0__:rpush", "inline:dst"], _subscriber} =
Client.read(subscriber)
Client.close(subscriber)
end
test "a blocked BLMOVE replays into the very same lists", %{conn: conn, port: port} do
waiter = blockingTask(port, ["BLMOVE", "replay:src", "replay:dst", "LEFT", "RIGHT", "0"])
awaitWaiters("replay:src", 1)
assert {2, conn} = Client.command(conn, ["RPUSH", "replay:src", "moved", "stays"])
assert Task.await(waiter) == "moved"
{source, conn} = Client.command(conn, ["LRANGE", "replay:src", "0", "-1"])
{destination, conn} = Client.command(conn, ["LRANGE", "replay:dst", "0", "-1"])
log = flushedLog()
refute log =~ "BLMOVE"
conn = replayFromScratch(conn)
assert {^source, conn} = Client.command(conn, ["LRANGE", "replay:src", "0", "-1"])
assert {^destination, _conn} = Client.command(conn, ["LRANGE", "replay:dst", "0", "-1"])
end
test "the pop of a woken BLPOP is logged after the push that woke it", %{
conn: conn,
port: port
} do
waiter = blockingTask(port, ["BLPOP", "order:queue", "0"])
awaitWaiters("order:queue", 1)
assert {1, conn} = Client.command(conn, ["LPUSH", "order:queue", "payload"])
assert Task.await(waiter) == ["order:queue", "payload"]
log = flushedLog()
assert offsetOf(log, "LPUSH") < offsetOf(log, "LPOP")
conn = replayFromScratch(conn)
assert {0, _conn} = Client.command(conn, ["EXISTS", "order:queue"])
end
end
describe "the wake signal belongs to the command layer" do
test "a push dispatched straight through the command router wakes a waiter", %{port: port} do
waiter = blockingTask(port, ["BLPOP", "hook:queue", "2"])
awaitWaiters("hook:queue", 1)
Command.execute(["LPUSH", "hook:queue", "direct"], commandState())
assert Task.await(waiter) == ["hook:queue", "direct"]
end
test "a rename dispatched straight through the command router wakes a waiter", %{port: port} do
assert {:ok, 1} = Store.rpush("hook:source", ["carried"])
waiter = blockingTask(port, ["BLPOP", "hook:target", "2"])
awaitWaiters("hook:target", 1)
Command.execute(["RENAME", "hook:source", "hook:target"], commandState())
assert Task.await(waiter) == ["hook:target", "carried"]
end
end
defp commandState do
Map.put(Excache.Command.Transaction.initial_state(), :subscriptions, MapSet.new())
end
defp fillOverLimit(conn, listKey) do
conn =
Enum.reduce(1..30, conn, fn index, acc ->
{{:simple, "OK"}, acc} = Client.command(acc, ["SET", "oom:key:#{index}", @payload])
acc
end)
{1, conn} = Client.command(conn, ["RPUSH", listKey, "value"])
{flat, conn} = Client.command(conn, ["MEMORY", "STATS"])
allocated =
flat
|> Enum.chunk_every(2)
|> Map.new(fn [name, value] -> {name, value} end)
|> Map.fetch!("total.allocated")
{{:simple, "OK"}, conn} =
Client.command(conn, ["CONFIG", "SET", "maxmemory", Integer.to_string(div(allocated, 2))])
conn
end
defp subscribeTo(port, channel) do
subscriber = Client.connect(port)
{["subscribe", ^channel, 1], subscriber} = Client.command(subscriber, ["SUBSCRIBE", channel])
subscriber
end
defp blockingTask(port, args) do
Task.async(fn ->
client = Client.connect(port)
{reply, _client} = Client.command(client, args)
Client.close(client)
reply
end)
end
defp awaitWaiters(key, count), do: awaitUntil(fn -> Blocking.waitingCount(key) == count end)
defp awaitUntil(check, attempts \\ 400) do
cond do
check.() ->
:ok
attempts == 0 ->
flunk("the expected registry state was never reached")
true ->
Process.sleep(5)
awaitUntil(check, attempts - 1)
end
end
defp offsetOf(log, needle) do
{offset, _length} = :binary.match(log, needle)
offset
end
defp flushedLog do
Aof.flush()
{:ok, log} = File.read(Application.get_env(:excache, :aof_path))
log
end
defp replayFromScratch(conn) do
Store.flushdb()
Aof.replay(Application.get_env(:excache, :aof_path))
conn
end
end