Merge pull request #1 from dualfroz/feat/pending-work-2026-09
feat: land pending September work
This commit is contained in:
36 files changed
+7849
-74
No files matched your search
@@ -11,8 +11,12 @@ defmodule Excache.Application do
|
|||||||
{Registry, keys: :duplicate, name: Excache.PubSub.Registry},
|
{Registry, keys: :duplicate, name: Excache.PubSub.Registry},
|
||||||
Excache.Store.Supervisor,
|
Excache.Store.Supervisor,
|
||||||
Excache.Aof,
|
Excache.Aof,
|
||||||
|
Excache.Blocking,
|
||||||
Excache.Expiry,
|
Excache.Expiry,
|
||||||
{DynamicSupervisor, strategy: :one_for_one, name: Excache.Server.ConnectionSupervisor}
|
trackerSpec(),
|
||||||
|
Excache.Eviction.Runner,
|
||||||
|
{DynamicSupervisor, strategy: :one_for_one, name: Excache.Server.ConnectionSupervisor},
|
||||||
|
Excache.Clients
|
||||||
]
|
]
|
||||||
|
|
||||||
case Supervisor.start_link(children, strategy: :one_for_one, name: Excache.Supervisor) do
|
case Supervisor.start_link(children, strategy: :one_for_one, name: Excache.Supervisor) do
|
||||||
@@ -25,6 +29,15 @@ defmodule Excache.Application do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def trackerSpec do
|
||||||
|
{Excache.Eviction.Tracker,
|
||||||
|
[
|
||||||
|
name: Excache.Eviction.Tracker,
|
||||||
|
logFactor: Excache.Config.lfuLogFactor(),
|
||||||
|
decayTime: Excache.Config.lfuDecayTime()
|
||||||
|
]}
|
||||||
|
end
|
||||||
|
|
||||||
defp start_acceptor(supervisor, port) when is_integer(port) do
|
defp start_acceptor(supervisor, port) when is_integer(port) do
|
||||||
case Supervisor.start_child(supervisor, {Excache.Server.Acceptor, [port: port]}) do
|
case Supervisor.start_child(supervisor, {Excache.Server.Acceptor, [port: port]}) do
|
||||||
{:ok, _pid} -> {:ok, supervisor}
|
{:ok, _pid} -> {:ok, supervisor}
|
||||||
|
|||||||
@@ -0,0 +1,280 @@
|
|||||||
|
defmodule Excache.Blocking do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
use GenServer
|
||||||
|
|
||||||
|
alias Excache.Store
|
||||||
|
alias Excache.Command.Reply
|
||||||
|
|
||||||
|
@name __MODULE__
|
||||||
|
|
||||||
|
def start_link(_arg), do: GenServer.start_link(__MODULE__, nil, name: @name)
|
||||||
|
|
||||||
|
def acquire(request), do: GenServer.call(@name, {:acquire, request, :block})
|
||||||
|
|
||||||
|
def attempt(request) do
|
||||||
|
{:served, term} = GenServer.call(@name, {:acquire, request, :immediate})
|
||||||
|
term
|
||||||
|
end
|
||||||
|
|
||||||
|
def signal([]), do: []
|
||||||
|
def signal(keys), do: GenServer.call(@name, {:ready, keys})
|
||||||
|
|
||||||
|
def takeWrites, do: GenServer.call(@name, :takeWrites)
|
||||||
|
|
||||||
|
def waitingCount(key), do: GenServer.call(@name, {:waitingCount, key})
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def init(_arg), do: {:ok, %{queues: %{}, waiters: %{}, monitors: %{}, writes: []}}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:acquire, request, mode}, {pid, _tag}, state) do
|
||||||
|
waiter = %{
|
||||||
|
ref: make_ref(),
|
||||||
|
pid: pid,
|
||||||
|
keys: request.keys,
|
||||||
|
op: request.op,
|
||||||
|
monitor: nil,
|
||||||
|
timer: nil
|
||||||
|
}
|
||||||
|
|
||||||
|
state = enqueue(state, waiter)
|
||||||
|
{state, served} = drain(state, request.keys, [])
|
||||||
|
|
||||||
|
case {deliver(served, waiter.ref), mode} do
|
||||||
|
{nil, :immediate} -> {:reply, {:served, missReply(request.op)}, dropWaiter(state, waiter)}
|
||||||
|
{nil, :block} -> {:reply, {:blocked, waiter.ref}, arm(state, waiter, request.timeout)}
|
||||||
|
{term, _mode} -> {:reply, {:served, term}, state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def handle_call({:ready, keys}, _from, state) do
|
||||||
|
{state, served} = drain(state, keys, [])
|
||||||
|
deliver(served, nil)
|
||||||
|
takeWrites(state)
|
||||||
|
end
|
||||||
|
|
||||||
|
def handle_call(:takeWrites, _from, state), do: takeWrites(state)
|
||||||
|
|
||||||
|
def handle_call({:waitingCount, key}, _from, state) do
|
||||||
|
count =
|
||||||
|
case Map.fetch(state.queues, key) do
|
||||||
|
{:ok, queue} -> :queue.len(queue)
|
||||||
|
:error -> 0
|
||||||
|
end
|
||||||
|
|
||||||
|
{:reply, count, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info({:blockingTimeout, ref}, state) do
|
||||||
|
case Map.fetch(state.waiters, ref) do
|
||||||
|
{:ok, waiter} ->
|
||||||
|
send(waiter.pid, {:excacheUnblocked, ref, {:array, nil}})
|
||||||
|
{:noreply, dropWaiter(state, waiter)}
|
||||||
|
|
||||||
|
:error ->
|
||||||
|
{:noreply, state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def handle_info({:DOWN, monitor, :process, _pid, _reason}, state) do
|
||||||
|
ref = Map.fetch!(state.monitors, monitor)
|
||||||
|
{:noreply, dropWaiter(state, Map.fetch!(state.waiters, ref))}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp drain(state, [], served), do: {state, served}
|
||||||
|
|
||||||
|
defp drain(state, [key | rest], served) do
|
||||||
|
{state, served, chained} = drainKey(state, key, served, [])
|
||||||
|
drain(state, chained ++ rest, served)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp drainKey(state, key, served, chained) do
|
||||||
|
case headWaiter(state, key) do
|
||||||
|
nil ->
|
||||||
|
{state, served, chained}
|
||||||
|
|
||||||
|
waiter ->
|
||||||
|
case serveWaiter(waiter) do
|
||||||
|
:empty ->
|
||||||
|
{state, served, chained}
|
||||||
|
|
||||||
|
{:ok, term, ready, writes} ->
|
||||||
|
state = recordWrites(dropWaiter(state, waiter), writes)
|
||||||
|
drainKey(state, key, [{waiter, term} | served], ready ++ chained)
|
||||||
|
|
||||||
|
{:error, term} ->
|
||||||
|
drainKey(dropWaiter(state, waiter), key, [{waiter, term} | served], chained)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp recordWrites(state, writes), do: %{state | writes: state.writes ++ writes}
|
||||||
|
|
||||||
|
defp takeWrites(state), do: {:reply, state.writes, %{state | writes: []}}
|
||||||
|
|
||||||
|
defp serveWaiter(waiter) do
|
||||||
|
Enum.reduce_while(waiter.keys, :empty, fn key, _pending ->
|
||||||
|
case serve(waiter.op, key) do
|
||||||
|
:empty -> {:cont, :empty}
|
||||||
|
outcome -> {:halt, outcome}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp serve({:pop, side}, key) do
|
||||||
|
case popOne(key, side) do
|
||||||
|
{:ok, value} ->
|
||||||
|
{:ok, {:array, [{:bulk, key}, {:bulk, value}]}, [], [{popName(side), [key]}]}
|
||||||
|
|
||||||
|
nil ->
|
||||||
|
:empty
|
||||||
|
|
||||||
|
{:error, reason} ->
|
||||||
|
{:error, errorTerm(reason)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp serve({:mpop, side, count}, key) do
|
||||||
|
case popMany(key, side, count) do
|
||||||
|
{:ok, [_ | _] = values} ->
|
||||||
|
taken = Integer.to_string(length(values))
|
||||||
|
|
||||||
|
{:ok, {:array, [{:bulk, key}, {:array, Reply.bulks(values)}]}, [],
|
||||||
|
[{popName(side), [key, taken]}]}
|
||||||
|
|
||||||
|
{:ok, []} ->
|
||||||
|
:empty
|
||||||
|
|
||||||
|
nil ->
|
||||||
|
:empty
|
||||||
|
|
||||||
|
{:error, reason} ->
|
||||||
|
{:error, errorTerm(reason)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp serve({:move, fromSide, toSide, destination}, key) do
|
||||||
|
case popOne(key, fromSide) do
|
||||||
|
{:ok, value} ->
|
||||||
|
moveInto(key, destination, fromSide, toSide, value)
|
||||||
|
|
||||||
|
nil ->
|
||||||
|
:empty
|
||||||
|
|
||||||
|
{:error, reason} ->
|
||||||
|
{:error, errorTerm(reason)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp moveInto(source, destination, fromSide, toSide, value) do
|
||||||
|
case pushOne(destination, toSide, value) do
|
||||||
|
{:ok, _length} ->
|
||||||
|
{:ok, {:bulk, value}, [destination],
|
||||||
|
moveWrites(source, destination, fromSide, toSide, value)}
|
||||||
|
|
||||||
|
{:error, reason} ->
|
||||||
|
pushOne(source, fromSide, value)
|
||||||
|
{:error, errorTerm(reason)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp popOne(key, :left), do: Store.lpop(key, nil)
|
||||||
|
defp popOne(key, :right), do: Store.rpop(key, nil)
|
||||||
|
|
||||||
|
defp popMany(key, :left, count), do: Store.lpop(key, count)
|
||||||
|
defp popMany(key, :right, count), do: Store.rpop(key, count)
|
||||||
|
|
||||||
|
defp pushOne(key, :left, value), do: Store.lpush(key, [value])
|
||||||
|
defp pushOne(key, :right, value), do: Store.rpush(key, [value])
|
||||||
|
|
||||||
|
defp popName(:left), do: "LPOP"
|
||||||
|
defp popName(:right), do: "RPOP"
|
||||||
|
|
||||||
|
defp pushName(:left), do: "LPUSH"
|
||||||
|
defp pushName(:right), do: "RPUSH"
|
||||||
|
|
||||||
|
defp moveWrites(source, destination, :right, :left, _value),
|
||||||
|
do: [{"RPOPLPUSH", [source, destination]}]
|
||||||
|
|
||||||
|
defp moveWrites(source, destination, fromSide, toSide, value),
|
||||||
|
do: [{popName(fromSide), [source]}, {pushName(toSide), [destination, value]}]
|
||||||
|
|
||||||
|
defp missReply({:move, _fromSide, _toSide, _destination}), do: {:bulk, nil}
|
||||||
|
defp missReply(_op), do: {:array, nil}
|
||||||
|
|
||||||
|
defp errorTerm(:wrongtype), do: Reply.wrongtype()
|
||||||
|
|
||||||
|
defp deliver(served, inlineRef) do
|
||||||
|
Enum.reduce(served, nil, fn {waiter, term}, inline ->
|
||||||
|
if waiter.ref == inlineRef do
|
||||||
|
term
|
||||||
|
else
|
||||||
|
send(waiter.pid, {:excacheUnblocked, waiter.ref, term})
|
||||||
|
inline
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp headWaiter(state, key) do
|
||||||
|
with {:ok, queue} <- Map.fetch(state.queues, key),
|
||||||
|
{:value, ref} <- :queue.peek(queue) do
|
||||||
|
Map.fetch!(state.waiters, ref)
|
||||||
|
else
|
||||||
|
_missing -> nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp enqueue(state, waiter) do
|
||||||
|
queues =
|
||||||
|
Enum.reduce(Enum.uniq(waiter.keys), state.queues, fn key, acc ->
|
||||||
|
Map.put(acc, key, :queue.in(waiter.ref, Map.get(acc, key, :queue.new())))
|
||||||
|
end)
|
||||||
|
|
||||||
|
%{state | queues: queues, waiters: Map.put(state.waiters, waiter.ref, waiter)}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp dropWaiter(state, waiter) do
|
||||||
|
armed = Map.fetch!(state.waiters, waiter.ref)
|
||||||
|
if armed.monitor, do: Process.demonitor(armed.monitor, [:flush])
|
||||||
|
if armed.timer, do: Process.cancel_timer(armed.timer)
|
||||||
|
|
||||||
|
queues =
|
||||||
|
Enum.reduce(Enum.uniq(armed.keys), state.queues, fn key, acc ->
|
||||||
|
dropFromQueue(acc, key, armed.ref)
|
||||||
|
end)
|
||||||
|
|
||||||
|
%{
|
||||||
|
state
|
||||||
|
| queues: queues,
|
||||||
|
waiters: Map.delete(state.waiters, armed.ref),
|
||||||
|
monitors: Map.delete(state.monitors, armed.monitor)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp dropFromQueue(queues, key, ref) do
|
||||||
|
pruned = :queue.filter(&(&1 != ref), Map.fetch!(queues, key))
|
||||||
|
|
||||||
|
if :queue.is_empty(pruned) do
|
||||||
|
Map.delete(queues, key)
|
||||||
|
else
|
||||||
|
Map.put(queues, key, pruned)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp arm(state, waiter, timeout) do
|
||||||
|
monitor = Process.monitor(waiter.pid)
|
||||||
|
|
||||||
|
timer =
|
||||||
|
if timeout > 0, do: Process.send_after(self(), {:blockingTimeout, waiter.ref}, timeout)
|
||||||
|
|
||||||
|
armed = %{waiter | monitor: monitor, timer: timer}
|
||||||
|
|
||||||
|
%{
|
||||||
|
state
|
||||||
|
| waiters: Map.put(state.waiters, waiter.ref, armed),
|
||||||
|
monitors: Map.put(state.monitors, monitor, waiter.ref)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
defmodule Excache.Clients do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
use GenServer
|
||||||
|
|
||||||
|
@name __MODULE__
|
||||||
|
@supervisor Excache.Server.ConnectionSupervisor
|
||||||
|
@pubsubRegistry Excache.PubSub.Registry
|
||||||
|
@defaultPoll 500
|
||||||
|
@selfKillDelay 50
|
||||||
|
|
||||||
|
def start_link(_arg), do: GenServer.start_link(__MODULE__, nil, name: @name)
|
||||||
|
|
||||||
|
def snapshot, do: GenServer.call(@name, :snapshot)
|
||||||
|
|
||||||
|
def find(pid), do: GenServer.call(@name, {:find, pid})
|
||||||
|
|
||||||
|
def setName(pid, name), do: GenServer.call(@name, {:setName, pid, name})
|
||||||
|
|
||||||
|
def kill(matcher, caller), do: GenServer.call(@name, {:kill, matcher, caller})
|
||||||
|
|
||||||
|
def subscriptionCount(pid), do: length(Registry.keys(@pubsubRegistry, pid))
|
||||||
|
|
||||||
|
def subscribedChannels do
|
||||||
|
@pubsubRegistry
|
||||||
|
|> Registry.select([{{:"$1", :_, :_}, [], [:"$1"]}])
|
||||||
|
|> Enum.uniq()
|
||||||
|
|> length()
|
||||||
|
end
|
||||||
|
|
||||||
|
def blocked?(pid) do
|
||||||
|
case Process.info(pid, :monitored_by) do
|
||||||
|
{:monitored_by, watchers} -> Process.whereis(Excache.Blocking) in watchers
|
||||||
|
nil -> false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def init(_arg) do
|
||||||
|
schedule()
|
||||||
|
|
||||||
|
{:ok,
|
||||||
|
%{
|
||||||
|
nextId: 1,
|
||||||
|
clients: %{},
|
||||||
|
monitors: %{},
|
||||||
|
connectionsReceived: 0,
|
||||||
|
closedNetIn: 0,
|
||||||
|
closedNetOut: 0,
|
||||||
|
runId: newRunId(),
|
||||||
|
startedAt: System.system_time(:millisecond)
|
||||||
|
}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:snapshot, _from, state) do
|
||||||
|
state = sync(state)
|
||||||
|
{:reply, buildSnapshot(state), state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:find, pid}, _from, state) do
|
||||||
|
state = sync(state)
|
||||||
|
{:reply, Map.get(state.clients, pid), state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:setName, pid, name}, _from, state) do
|
||||||
|
state = sync(state)
|
||||||
|
|
||||||
|
case Map.fetch(state.clients, pid) do
|
||||||
|
{:ok, client} ->
|
||||||
|
renamed = Map.put(state.clients, pid, %{client | name: name})
|
||||||
|
{:reply, :ok, %{state | clients: renamed}}
|
||||||
|
|
||||||
|
:error ->
|
||||||
|
{:reply, :error, state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:kill, matcher, caller}, _from, state) do
|
||||||
|
state = sync(state)
|
||||||
|
victims = state.clients |> Map.values() |> Enum.filter(matcher)
|
||||||
|
{itself, others} = Enum.split_with(victims, &(&1.pid == caller))
|
||||||
|
|
||||||
|
Enum.each(others, fn client -> terminate(client.pid) end)
|
||||||
|
|
||||||
|
Enum.each(itself, fn client ->
|
||||||
|
Process.send_after(self(), {:terminate, client.pid}, @selfKillDelay)
|
||||||
|
end)
|
||||||
|
|
||||||
|
{:reply, length(victims), state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info(:poll, state) do
|
||||||
|
schedule()
|
||||||
|
{:noreply, sync(state)}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info({:terminate, pid}, state) do
|
||||||
|
terminate(pid)
|
||||||
|
{:noreply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info({:DOWN, monitor, :process, pid, _reason}, state) do
|
||||||
|
{_pid, monitors} = Map.pop(state.monitors, monitor)
|
||||||
|
{client, clients} = Map.pop(state.clients, pid)
|
||||||
|
{:noreply, accountClosed(%{state | monitors: monitors, clients: clients}, client)}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp schedule do
|
||||||
|
interval = Application.get_env(:excache, :clientsPollInterval, @defaultPoll)
|
||||||
|
Process.send_after(self(), :poll, interval)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp sync(state) do
|
||||||
|
Enum.reduce(livePids(), state, fn pid, acc ->
|
||||||
|
case Map.fetch(acc.clients, pid) do
|
||||||
|
{:ok, client} -> %{acc | clients: Map.put(acc.clients, pid, refresh(client))}
|
||||||
|
:error -> register(acc, pid)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp livePids do
|
||||||
|
for {_id, pid, _type, _modules} <- DynamicSupervisor.which_children(@supervisor),
|
||||||
|
is_pid(pid),
|
||||||
|
do: pid
|
||||||
|
end
|
||||||
|
|
||||||
|
defp register(state, pid) do
|
||||||
|
with socket when socket != nil <- socketOf(pid),
|
||||||
|
{netIn, netOut} <- netStats(socket) do
|
||||||
|
now = System.system_time(:millisecond)
|
||||||
|
|
||||||
|
client = %{
|
||||||
|
id: state.nextId,
|
||||||
|
pid: pid,
|
||||||
|
socket: socket,
|
||||||
|
addr: addressOf(socket, &:inet.peername/1),
|
||||||
|
laddr: addressOf(socket, &:inet.sockname/1),
|
||||||
|
name: "",
|
||||||
|
connectedAt: now,
|
||||||
|
lastActiveAt: now,
|
||||||
|
netIn: netIn,
|
||||||
|
netOut: netOut
|
||||||
|
}
|
||||||
|
|
||||||
|
%{
|
||||||
|
state
|
||||||
|
| nextId: state.nextId + 1,
|
||||||
|
clients: Map.put(state.clients, pid, client),
|
||||||
|
monitors: Map.put(state.monitors, Process.monitor(pid), pid),
|
||||||
|
connectionsReceived: state.connectionsReceived + 1
|
||||||
|
}
|
||||||
|
else
|
||||||
|
_unavailable -> state
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp refresh(client) do
|
||||||
|
case netStats(client.socket) do
|
||||||
|
{netIn, netOut} when netIn > client.netIn ->
|
||||||
|
%{
|
||||||
|
client
|
||||||
|
| netIn: netIn,
|
||||||
|
netOut: netOut,
|
||||||
|
lastActiveAt: System.system_time(:millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
{netIn, netOut} ->
|
||||||
|
%{client | netIn: netIn, netOut: netOut}
|
||||||
|
|
||||||
|
:closed ->
|
||||||
|
client
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp accountClosed(state, nil), do: state
|
||||||
|
|
||||||
|
defp accountClosed(state, client) do
|
||||||
|
%{
|
||||||
|
state
|
||||||
|
| closedNetIn: state.closedNetIn + client.netIn,
|
||||||
|
closedNetOut: state.closedNetOut + client.netOut
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp buildSnapshot(state) do
|
||||||
|
clients = state.clients |> Map.values() |> Enum.sort_by(& &1.id)
|
||||||
|
|
||||||
|
%{
|
||||||
|
clients: clients,
|
||||||
|
connectionsReceived: state.connectionsReceived,
|
||||||
|
netIn: Enum.reduce(clients, state.closedNetIn, fn client, sum -> sum + client.netIn end),
|
||||||
|
netOut: Enum.reduce(clients, state.closedNetOut, fn client, sum -> sum + client.netOut end),
|
||||||
|
runId: state.runId,
|
||||||
|
startedAt: state.startedAt
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp terminate(pid), do: DynamicSupervisor.terminate_child(@supervisor, pid)
|
||||||
|
|
||||||
|
defp socketOf(pid) do
|
||||||
|
case Process.info(pid, :links) do
|
||||||
|
{:links, links} -> Enum.find(links, &tcpPort?/1)
|
||||||
|
nil -> nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp tcpPort?(link) when is_port(link) do
|
||||||
|
match?({:name, ~c"tcp_inet"}, Port.info(link, :name))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp tcpPort?(_link), do: false
|
||||||
|
|
||||||
|
defp netStats(socket) do
|
||||||
|
case :inet.getstat(socket, [:recv_oct, :send_oct]) do
|
||||||
|
{:ok, [recv_oct: netIn, send_oct: netOut]} -> {netIn, netOut}
|
||||||
|
{:error, _reason} -> :closed
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp addressOf(socket, fetch) do
|
||||||
|
case fetch.(socket) do
|
||||||
|
{:ok, {ip, port}} -> formatAddress(ip, port)
|
||||||
|
{:error, _reason} -> ""
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp formatAddress(ip, port) when tuple_size(ip) == 8, do: "[#{:inet.ntoa(ip)}]:#{port}"
|
||||||
|
defp formatAddress(ip, port), do: "#{:inet.ntoa(ip)}:#{port}"
|
||||||
|
|
||||||
|
defp newRunId, do: 20 |> :rand.bytes() |> Base.encode16(case: :lower)
|
||||||
|
end
|
||||||
+79
-12
@@ -1,8 +1,9 @@
|
|||||||
defmodule Excache.Command do
|
defmodule Excache.Command do
|
||||||
@moduledoc false
|
@moduledoc false
|
||||||
|
|
||||||
alias Excache.{PubSub, Store, Persistence, Aof, Notify}
|
alias Excache.{Blocking, PubSub, Store, Persistence, Aof, Notify}
|
||||||
alias Excache.Command.{Reply, Transaction}
|
alias Excache.Command.{Reply, Transaction}
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
|
||||||
import Reply,
|
import Reply,
|
||||||
only: [
|
only: [
|
||||||
@@ -45,10 +46,25 @@ defmodule Excache.Command do
|
|||||||
ZADD ZREM ZINCRBY ZPOPMIN ZPOPMAX
|
ZADD ZREM ZINCRBY ZPOPMIN ZPOPMAX
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@oomDenied ~w(
|
||||||
|
SET SETNX SETRANGE APPEND GETSET MSET INCR INCRBY DECR DECRBY INCRBYFLOAT
|
||||||
|
LPUSH RPUSH LINSERT LSET RPOPLPUSH BLMOVE BRPOPLPUSH
|
||||||
|
HSET HSETNX HINCRBY
|
||||||
|
SADD SMOVE SINTERSTORE SUNIONSTORE SDIFFSTORE
|
||||||
|
ZADD ZINCRBY
|
||||||
|
XADD XSETID XGROUP XREADGROUP XCLAIM XAUTOCLAIM
|
||||||
|
SETBIT BITOP BITFIELD PFADD PFMERGE
|
||||||
|
COPY RENAME RENAMENX
|
||||||
|
)
|
||||||
|
|
||||||
|
@oom {:error, "OOM command not allowed when used memory > 'maxmemory'"}
|
||||||
|
|
||||||
def known_commands, do: @known ++ Excache.Command.Extended.command_names()
|
def known_commands, do: @known ++ Excache.Command.Extended.command_names()
|
||||||
|
|
||||||
@extended_write Excache.Command.Extended.write_command_names()
|
@extended_write Excache.Command.Extended.write_command_names()
|
||||||
|
|
||||||
|
@blockingNames Excache.Command.Blocking.commandNames()
|
||||||
|
|
||||||
def execute([], state), do: {[], state}
|
def execute([], state), do: {[], state}
|
||||||
|
|
||||||
def execute([name | args], state) do
|
def execute([name | args], state) do
|
||||||
@@ -61,27 +77,76 @@ defmodule Excache.Command do
|
|||||||
end
|
end
|
||||||
|
|
||||||
def dispatch_and_log(command, args, state) do
|
def dispatch_and_log(command, args, state) do
|
||||||
{replies, updated} = dispatch(command, args, state)
|
if oomDenied?(command) and Runner.admit() == :oom do
|
||||||
|
{[@oom], state}
|
||||||
if writeCommand?(command) and not Enum.any?(replies, &match?({:error, _}, &1)) do
|
else
|
||||||
Aof.append(command, aofArgs(command, args, replies))
|
logged(command, args, state)
|
||||||
Notify.for_command(command, args)
|
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp logged(command, args, state) do
|
||||||
|
{replies, updated} = dispatch(command, args, state)
|
||||||
|
succeeded = not Enum.any?(replies, &match?({:error, _}, &1))
|
||||||
|
|
||||||
|
if succeeded and oomDenied?(command), do: Runner.account(args)
|
||||||
|
|
||||||
|
entries = if succeeded, do: aofEntries(command, args, replies), else: []
|
||||||
|
logWrites(entries)
|
||||||
|
logWrites(servedWrites(command, wakeKeys(entries)))
|
||||||
|
|
||||||
{replies, updated}
|
{replies, updated}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp logWrites(entries) do
|
||||||
|
Enum.each(entries, fn {name, args} ->
|
||||||
|
if writeCommand?(name) do
|
||||||
|
Aof.append(name, args)
|
||||||
|
Notify.for_command(name, args)
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp servedWrites(command, keys) do
|
||||||
|
if command in @blockingNames, do: Blocking.takeWrites(), else: Blocking.signal(keys)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp wakeKeys(entries), do: Enum.flat_map(entries, &listTargets/1)
|
||||||
|
|
||||||
|
defp listTargets({"LPUSH", [key | _values]}), do: [key]
|
||||||
|
defp listTargets({"RPUSH", [key | _values]}), do: [key]
|
||||||
|
defp listTargets({"LINSERT", [key | _rest]}), do: [key]
|
||||||
|
defp listTargets({"RPOPLPUSH", [_source, destination]}), do: [destination]
|
||||||
|
defp listTargets({"RENAME", [_source, destination]}), do: [destination]
|
||||||
|
defp listTargets({"RENAMENX", [_source, destination]}), do: [destination]
|
||||||
|
defp listTargets({"COPY", [_source, destination | _options]}), do: [destination]
|
||||||
|
defp listTargets(_entry), do: []
|
||||||
|
|
||||||
defp writeCommand?(command), do: command in @write_commands or command in @extended_write
|
defp writeCommand?(command), do: command in @write_commands or command in @extended_write
|
||||||
|
|
||||||
defp aofArgs("XADD", args, [{:bulk, id}]) when is_binary(id),
|
defp oomDenied?(command), do: command in @oomDenied
|
||||||
do: Excache.Command.Streams.rewriteAddId(args, id)
|
|
||||||
|
|
||||||
defp aofArgs(_command, args, _replies), do: args
|
defp aofEntries("XADD", args, [{:bulk, id}]) when is_binary(id),
|
||||||
|
do: [{"XADD", Excache.Command.Streams.rewriteAddId(args, id)}]
|
||||||
|
|
||||||
|
defp aofEntries("SPOP", [key | _rest] = args, replies) do
|
||||||
|
case poppedMembers(replies) do
|
||||||
|
[] -> [{"SPOP", args}]
|
||||||
|
members -> [{"SREM", [key | members]}]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp aofEntries(command, args, _replies), do: [{command, args}]
|
||||||
|
|
||||||
|
defp poppedMembers([{:bulk, member}]) when is_binary(member), do: [member]
|
||||||
|
|
||||||
|
defp poppedMembers([{:array, entries}]),
|
||||||
|
do: for({:bulk, member} <- entries, is_binary(member), do: member)
|
||||||
|
|
||||||
|
defp poppedMembers(_replies), do: []
|
||||||
|
|
||||||
defp dispatch("PING", [], state), do: {[{:simple, "PONG"}], state}
|
defp dispatch("PING", [], state), do: {[{:simple, "PONG"}], state}
|
||||||
defp dispatch("PING", [message], state), do: {[{:bulk, message}], state}
|
defp dispatch("PING", [message], state), do: {[{:bulk, message}], state}
|
||||||
defp dispatch("ECHO", [message], state), do: {[{:bulk, message}], state}
|
defp dispatch("ECHO", [message], state), do: {[{:bulk, message}], state}
|
||||||
defp dispatch("COMMAND", _args, state), do: {[{:array, []}], state}
|
|
||||||
defp dispatch("SELECT", [_db], state), do: {[{:simple, "OK"}], state}
|
defp dispatch("SELECT", [_db], state), do: {[{:simple, "OK"}], state}
|
||||||
defp dispatch("QUIT", [], state), do: {[{:simple, "OK"}], state}
|
defp dispatch("QUIT", [], state), do: {[{:simple, "OK"}], state}
|
||||||
defp dispatch("DBSIZE", [], state), do: {[{:integer, Store.dbsize()}], state}
|
defp dispatch("DBSIZE", [], state), do: {[{:integer, Store.dbsize()}], state}
|
||||||
@@ -287,13 +352,15 @@ defmodule Excache.Command do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp dispatch("ZPOPMIN", [key], state), do: reply(zrange_reply(Store.zpopmin(key, 1), true), state)
|
defp dispatch("ZPOPMIN", [key], state),
|
||||||
|
do: reply(zrange_reply(Store.zpopmin(key, 1), true), state)
|
||||||
|
|
||||||
defp dispatch("ZPOPMIN", [key, count], state) do
|
defp dispatch("ZPOPMIN", [key, count], state) do
|
||||||
with_int(count, state, fn n -> zrange_reply(Store.zpopmin(key, n), true) end)
|
with_int(count, state, fn n -> zrange_reply(Store.zpopmin(key, n), true) end)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp dispatch("ZPOPMAX", [key], state), do: reply(zrange_reply(Store.zpopmax(key, 1), true), state)
|
defp dispatch("ZPOPMAX", [key], state),
|
||||||
|
do: reply(zrange_reply(Store.zpopmax(key, 1), true), state)
|
||||||
|
|
||||||
defp dispatch("ZPOPMAX", [key, count], state) do
|
defp dispatch("ZPOPMAX", [key, count], state) do
|
||||||
with_int(count, state, fn n -> zrange_reply(Store.zpopmax(key, n), true) end)
|
with_int(count, state, fn n -> zrange_reply(Store.zpopmax(key, n), true) end)
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
defmodule Excache.Command.Blocking do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Command.Reply
|
||||||
|
|
||||||
|
@names ~w(BLPOP BRPOP BLMOVE BRPOPLPUSH BLMPOP)
|
||||||
|
|
||||||
|
def commandNames, do: @names
|
||||||
|
|
||||||
|
def dispatch("BLPOP", args, state), do: popCommand(args, :left, state)
|
||||||
|
def dispatch("BRPOP", args, state), do: popCommand(args, :right, state)
|
||||||
|
|
||||||
|
def dispatch("BLMOVE", [source, destination, fromRaw, toRaw, timeoutRaw], state) do
|
||||||
|
with {:ok, fromSide} <- parseSide(fromRaw),
|
||||||
|
{:ok, toSide} <- parseSide(toRaw) do
|
||||||
|
withTimeout(timeoutRaw, state, fn timeout ->
|
||||||
|
submit([source], {:move, fromSide, toSide, destination}, timeout, state)
|
||||||
|
end)
|
||||||
|
else
|
||||||
|
:error -> {[Reply.syntax()], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("BRPOPLPUSH", [source, destination, timeoutRaw], state) do
|
||||||
|
withTimeout(timeoutRaw, state, fn timeout ->
|
||||||
|
submit([source], {:move, :right, :left, destination}, timeout, state)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("BLMPOP", [timeoutRaw, numkeysRaw | rest], state) when rest != [] do
|
||||||
|
withTimeout(timeoutRaw, state, fn timeout ->
|
||||||
|
case parseMpop(numkeysRaw, rest) do
|
||||||
|
{:ok, keys, side, count} -> submit(keys, {:mpop, side, count}, timeout, state)
|
||||||
|
{:reply, term} -> {[term], state}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch(_name, _args, _state), do: :unhandled
|
||||||
|
|
||||||
|
defp popCommand(args, side, state) when length(args) >= 2 do
|
||||||
|
{keys, [timeoutRaw]} = Enum.split(args, length(args) - 1)
|
||||||
|
|
||||||
|
withTimeout(timeoutRaw, state, fn timeout ->
|
||||||
|
submit(keys, {:pop, side}, timeout, state)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp popCommand(_args, _side, _state), do: :unhandled
|
||||||
|
|
||||||
|
defp submit(keys, op, timeout, state) do
|
||||||
|
request = %{keys: keys, op: op, timeout: timeout}
|
||||||
|
|
||||||
|
if state.in_multi do
|
||||||
|
{[Excache.Blocking.attempt(request)], state}
|
||||||
|
else
|
||||||
|
case Excache.Blocking.acquire(request) do
|
||||||
|
{:served, term} -> {[term], state}
|
||||||
|
{:blocked, ref} -> {[{:blocked, ref}], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp withTimeout(raw, state, fun) do
|
||||||
|
case parseTimeout(raw) do
|
||||||
|
{:ok, timeout} -> fun.(timeout)
|
||||||
|
{:reply, term} -> {[term], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseTimeout(raw) do
|
||||||
|
case Reply.parse_score(raw) do
|
||||||
|
{:ok, seconds} when seconds < 0 -> {:reply, {:error, "ERR timeout is negative"}}
|
||||||
|
{:ok, seconds} -> {:ok, trunc(seconds * 1000)}
|
||||||
|
:error -> {:reply, {:error, "ERR timeout is not a float or out of range"}}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseSide(raw) do
|
||||||
|
case String.upcase(raw) do
|
||||||
|
"LEFT" -> {:ok, :left}
|
||||||
|
"RIGHT" -> {:ok, :right}
|
||||||
|
_other -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseMpop(numkeysRaw, rest) do
|
||||||
|
case Integer.parse(numkeysRaw) do
|
||||||
|
{numkeys, ""} when numkeys > 0 -> splitMpop(numkeys, rest)
|
||||||
|
_invalid -> {:reply, {:error, "ERR numkeys should be greater than 0"}}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp splitMpop(numkeys, rest) when length(rest) > numkeys do
|
||||||
|
{keys, [directionRaw | tail]} = Enum.split(rest, numkeys)
|
||||||
|
|
||||||
|
case parseSide(directionRaw) do
|
||||||
|
{:ok, side} -> mpopCount(keys, side, tail)
|
||||||
|
:error -> {:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp splitMpop(_numkeys, _rest), do: {:reply, Reply.syntax()}
|
||||||
|
|
||||||
|
defp mpopCount(keys, side, []), do: {:ok, keys, side, 1}
|
||||||
|
|
||||||
|
defp mpopCount(keys, side, [countToken, countRaw]) do
|
||||||
|
if String.upcase(countToken) == "COUNT" do
|
||||||
|
case Integer.parse(countRaw) do
|
||||||
|
{count, ""} when count > 0 -> {:ok, keys, side, count}
|
||||||
|
_invalid -> {:reply, {:error, "ERR count should be greater than 0"}}
|
||||||
|
end
|
||||||
|
else
|
||||||
|
{:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp mpopCount(_keys, _side, _tail), do: {:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
defmodule Excache.Command.Extended do
|
defmodule Excache.Command.Extended do
|
||||||
@moduledoc false
|
@moduledoc false
|
||||||
|
|
||||||
alias Excache.Command.{Streams, Bitmaps, HyperLogLog, Keyspace}
|
alias Excache.Command.{Streams, Bitmaps, HyperLogLog, Keyspace, Server, Blocking, Introspection}
|
||||||
|
|
||||||
@stream ~w(XADD XLEN XRANGE XREVRANGE XDEL XTRIM XSETID XREAD XINFO XGROUP
|
@stream ~w(XADD XLEN XRANGE XREVRANGE XDEL XTRIM XSETID XREAD XINFO XGROUP
|
||||||
XREADGROUP XACK XPENDING XCLAIM XAUTOCLAIM)
|
XREADGROUP XACK XPENDING XCLAIM XAUTOCLAIM)
|
||||||
@@ -17,17 +17,27 @@ defmodule Excache.Command.Extended do
|
|||||||
ZCOUNT ZREVRANGE ZREVRANK ZMSCORE
|
ZCOUNT ZREVRANGE ZREVRANK ZMSCORE
|
||||||
RENAME RENAMENX COPY TOUCH UNLINK EXPIREAT EXPIRETIME PEXPIRETIME)
|
RENAME RENAMENX COPY TOUCH UNLINK EXPIREAT EXPIRETIME PEXPIRETIME)
|
||||||
|
|
||||||
|
@server ~w(MEMORY CONFIG)
|
||||||
|
|
||||||
|
@blocking Excache.Command.Blocking.commandNames()
|
||||||
|
|
||||||
|
@introspection Excache.Command.Introspection.commandNames()
|
||||||
|
|
||||||
def dispatch(name, args, state) do
|
def dispatch(name, args, state) do
|
||||||
cond do
|
cond do
|
||||||
name in @stream -> Streams.dispatch(name, args, state)
|
name in @stream -> Streams.dispatch(name, args, state)
|
||||||
name in @bitmap -> Bitmaps.dispatch(name, args, state)
|
name in @bitmap -> Bitmaps.dispatch(name, args, state)
|
||||||
name in @hll -> HyperLogLog.dispatch(name, args, state)
|
name in @hll -> HyperLogLog.dispatch(name, args, state)
|
||||||
name in @keyspace -> Keyspace.dispatch(name, args, state)
|
name in @keyspace -> Keyspace.dispatch(name, args, state)
|
||||||
|
name in @server -> Server.dispatch(name, args, state)
|
||||||
|
name in @blocking -> Blocking.dispatch(name, args, state)
|
||||||
|
name in @introspection -> Introspection.dispatch(name, args, state)
|
||||||
true -> :unhandled
|
true -> :unhandled
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def command_names, do: @stream ++ @bitmap ++ @hll ++ @keyspace
|
def command_names,
|
||||||
|
do: @stream ++ @bitmap ++ @hll ++ @keyspace ++ @server ++ @blocking ++ @introspection
|
||||||
|
|
||||||
def write_command_names do
|
def write_command_names do
|
||||||
~w(XADD XDEL XTRIM XSETID XGROUP XREADGROUP XACK XCLAIM XAUTOCLAIM
|
~w(XADD XDEL XTRIM XSETID XGROUP XREADGROUP XACK XCLAIM XAUTOCLAIM
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
defmodule Excache.Command.Introspection do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Command.Introspection.Catalog
|
||||||
|
alias Excache.Command.Introspection.Client
|
||||||
|
alias Excache.Command.Introspection.Info
|
||||||
|
|
||||||
|
@names ~w(INFO CLIENT COMMAND)
|
||||||
|
|
||||||
|
def commandNames, do: @names
|
||||||
|
|
||||||
|
def dispatch("INFO", args, state), do: {[{:bulk, Info.render(args)}], state}
|
||||||
|
|
||||||
|
def dispatch("CLIENT", [subcommand | rest], state),
|
||||||
|
do: Client.dispatch(String.upcase(subcommand), rest, state)
|
||||||
|
|
||||||
|
def dispatch("CLIENT", [], state), do: {[wrongArity("client")], state}
|
||||||
|
|
||||||
|
def dispatch("COMMAND", [], state), do: {[fullTable()], state}
|
||||||
|
|
||||||
|
def dispatch("COMMAND", [subcommand | rest], state),
|
||||||
|
do: command(String.upcase(subcommand), rest, state)
|
||||||
|
|
||||||
|
defp command("COUNT", [], state), do: {[{:integer, Catalog.count()}], state}
|
||||||
|
|
||||||
|
defp command("LIST", [], state) do
|
||||||
|
{[{:array, Enum.map(Catalog.names(), fn name -> {:bulk, String.downcase(name)} end)}], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp command("INFO", [], state), do: {[fullTable()], state}
|
||||||
|
|
||||||
|
defp command("INFO", names, state) do
|
||||||
|
{[{:array, Enum.map(names, fn name -> Catalog.infoTerm(String.upcase(name)) end)}], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp command("DOCS", [], state) do
|
||||||
|
{[{:array, Enum.flat_map(Catalog.names(), &docsEntry/1)}], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp command("DOCS", names, state) do
|
||||||
|
entries =
|
||||||
|
names
|
||||||
|
|> Enum.map(&String.upcase/1)
|
||||||
|
|> Enum.filter(&Catalog.known?/1)
|
||||||
|
|> Enum.flat_map(&docsEntry/1)
|
||||||
|
|
||||||
|
{[{:array, entries}], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp command("GETKEYS", [name | args], state) do
|
||||||
|
{[getkeys(String.upcase(name), args)], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp command("GETKEYS", [], state), do: {[wrongArity("command|getkeys")], state}
|
||||||
|
|
||||||
|
defp command(subcommand, _args, state) do
|
||||||
|
{[
|
||||||
|
{:error,
|
||||||
|
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
|
||||||
|
"Try COMMAND HELP."}
|
||||||
|
], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fullTable, do: {:array, Enum.map(Catalog.names(), &Catalog.infoTerm/1)}
|
||||||
|
|
||||||
|
defp docsEntry(name), do: [{:bulk, String.downcase(name)}, Catalog.docsTerm(name)]
|
||||||
|
|
||||||
|
defp getkeys(name, args) do
|
||||||
|
cond do
|
||||||
|
not Catalog.known?(name) ->
|
||||||
|
{:error, "ERR Invalid command specified"}
|
||||||
|
|
||||||
|
not Catalog.arityOk?(name, length(args) + 1) ->
|
||||||
|
{:error, "ERR Invalid number of arguments specified for command"}
|
||||||
|
|
||||||
|
true ->
|
||||||
|
extractKeys(name, args)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp extractKeys(name, args) do
|
||||||
|
case Catalog.keysFor(name, args) do
|
||||||
|
{:ok, []} -> {:error, "ERR The command has no key arguments"}
|
||||||
|
{:ok, keys} -> {:array, Enum.map(keys, fn key -> {:bulk, key} end)}
|
||||||
|
:error -> {:error, "ERR Invalid arguments specified for command"}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp wrongArity(name),
|
||||||
|
do: {:error, "ERR wrong number of arguments for '#{name}' command"}
|
||||||
|
end
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
defmodule Excache.Command.Introspection.Catalog do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
# {arity, flags, firstKey, lastKey, step, group}
|
||||||
|
# Arity counts the command name, negative means "at least this many".
|
||||||
|
# firstKey/lastKey/step are 1 based positions in the full argument vector,
|
||||||
|
# a negative lastKey counts back from the end, zeroes mean "takes no keys".
|
||||||
|
|
||||||
|
@commands %{
|
||||||
|
"PING" => {-1, [:fast], 0, 0, 0, :connection},
|
||||||
|
"ECHO" => {2, [:fast], 0, 0, 0, :connection},
|
||||||
|
"QUIT" => {-1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :connection},
|
||||||
|
"SELECT" => {2, [:loading, :stale, :fast], 0, 0, 0, :connection},
|
||||||
|
"CLIENT" => {-2, [:admin, :noscript, :loading, :stale], 0, 0, 0, :connection},
|
||||||
|
"COMMAND" => {-1, [:loading, :stale], 0, 0, 0, :server},
|
||||||
|
"INFO" => {-1, [:loading, :stale], 0, 0, 0, :server},
|
||||||
|
"CONFIG" => {-2, [:admin, :noscript, :loading, :stale], 0, 0, 0, :server},
|
||||||
|
"MEMORY" => {-2, [:readonly], 0, 0, 0, :server},
|
||||||
|
"DBSIZE" => {1, [:readonly, :fast], 0, 0, 0, :server},
|
||||||
|
"FLUSHDB" => {-1, [:write], 0, 0, 0, :server},
|
||||||
|
"SAVE" => {1, [:admin, :noscript], 0, 0, 0, :server},
|
||||||
|
"BGSAVE" => {-1, [:admin, :noscript], 0, 0, 0, :server},
|
||||||
|
"GET" => {2, [:readonly, :fast], 1, 1, 1, :string},
|
||||||
|
"SET" => {-3, [:write, :denyoom], 1, 1, 1, :string},
|
||||||
|
"SETNX" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"GETSET" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"GETDEL" => {2, [:write, :fast], 1, 1, 1, :string},
|
||||||
|
"APPEND" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"STRLEN" => {2, [:readonly, :fast], 1, 1, 1, :string},
|
||||||
|
"SETRANGE" => {4, [:write, :denyoom], 1, 1, 1, :string},
|
||||||
|
"GETRANGE" => {4, [:readonly], 1, 1, 1, :string},
|
||||||
|
"INCR" => {2, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"DECR" => {2, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"INCRBY" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"DECRBY" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"INCRBYFLOAT" => {3, [:write, :denyoom, :fast], 1, 1, 1, :string},
|
||||||
|
"MSET" => {-3, [:write, :denyoom], 1, -1, 2, :string},
|
||||||
|
"MGET" => {-2, [:readonly, :fast], 1, -1, 1, :string},
|
||||||
|
"DEL" => {-2, [:write], 1, -1, 1, :generic},
|
||||||
|
"UNLINK" => {-2, [:write, :fast], 1, -1, 1, :generic},
|
||||||
|
"EXISTS" => {-2, [:readonly, :fast], 1, -1, 1, :generic},
|
||||||
|
"TOUCH" => {-2, [:readonly, :fast], 1, -1, 1, :generic},
|
||||||
|
"KEYS" => {2, [:readonly], 0, 0, 0, :generic},
|
||||||
|
"SCAN" => {-2, [:readonly], 0, 0, 0, :generic},
|
||||||
|
"TYPE" => {2, [:readonly, :fast], 1, 1, 1, :generic},
|
||||||
|
"OBJECT" => {-2, [:readonly], 0, 0, 0, :generic},
|
||||||
|
"RENAME" => {3, [:write], 1, 2, 1, :generic},
|
||||||
|
"RENAMENX" => {3, [:write, :fast], 1, 2, 1, :generic},
|
||||||
|
"COPY" => {-3, [:write, :denyoom], 1, 2, 1, :generic},
|
||||||
|
"EXPIRE" => {-3, [:write, :fast], 1, 1, 1, :generic},
|
||||||
|
"PEXPIRE" => {-3, [:write, :fast], 1, 1, 1, :generic},
|
||||||
|
"EXPIREAT" => {-3, [:write, :fast], 1, 1, 1, :generic},
|
||||||
|
"EXPIRETIME" => {2, [:readonly, :fast], 1, 1, 1, :generic},
|
||||||
|
"PEXPIRETIME" => {2, [:readonly, :fast], 1, 1, 1, :generic},
|
||||||
|
"TTL" => {2, [:readonly, :fast], 1, 1, 1, :generic},
|
||||||
|
"PTTL" => {2, [:readonly, :fast], 1, 1, 1, :generic},
|
||||||
|
"PERSIST" => {2, [:write, :fast], 1, 1, 1, :generic},
|
||||||
|
"LPUSH" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :list},
|
||||||
|
"RPUSH" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :list},
|
||||||
|
"LPOP" => {-2, [:write, :fast], 1, 1, 1, :list},
|
||||||
|
"RPOP" => {-2, [:write, :fast], 1, 1, 1, :list},
|
||||||
|
"LRANGE" => {4, [:readonly], 1, 1, 1, :list},
|
||||||
|
"LLEN" => {2, [:readonly, :fast], 1, 1, 1, :list},
|
||||||
|
"LINDEX" => {3, [:readonly], 1, 1, 1, :list},
|
||||||
|
"LSET" => {4, [:write, :denyoom], 1, 1, 1, :list},
|
||||||
|
"LREM" => {4, [:write], 1, 1, 1, :list},
|
||||||
|
"LTRIM" => {4, [:write], 1, 1, 1, :list},
|
||||||
|
"LINSERT" => {5, [:write, :denyoom], 1, 1, 1, :list},
|
||||||
|
"LPOS" => {-3, [:readonly], 1, 1, 1, :list},
|
||||||
|
"RPOPLPUSH" => {3, [:write, :denyoom], 1, 2, 1, :list},
|
||||||
|
"BLPOP" => {-3, [:write, :noscript, :blocking], 1, -2, 1, :list},
|
||||||
|
"BRPOP" => {-3, [:write, :noscript, :blocking], 1, -2, 1, :list},
|
||||||
|
"BLMOVE" => {6, [:write, :denyoom, :noscript, :blocking], 1, 2, 1, :list},
|
||||||
|
"BRPOPLPUSH" => {4, [:write, :denyoom, :noscript, :blocking], 1, 2, 1, :list},
|
||||||
|
"BLMPOP" => {-5, [:write, :noscript, :blocking, :movablekeys], 0, 0, 0, :list},
|
||||||
|
"HSET" => {-4, [:write, :denyoom, :fast], 1, 1, 1, :hash},
|
||||||
|
"HSETNX" => {4, [:write, :denyoom, :fast], 1, 1, 1, :hash},
|
||||||
|
"HGET" => {3, [:readonly, :fast], 1, 1, 1, :hash},
|
||||||
|
"HMGET" => {-3, [:readonly, :fast], 1, 1, 1, :hash},
|
||||||
|
"HDEL" => {-3, [:write, :fast], 1, 1, 1, :hash},
|
||||||
|
"HGETALL" => {2, [:readonly], 1, 1, 1, :hash},
|
||||||
|
"HKEYS" => {2, [:readonly], 1, 1, 1, :hash},
|
||||||
|
"HVALS" => {2, [:readonly], 1, 1, 1, :hash},
|
||||||
|
"HLEN" => {2, [:readonly, :fast], 1, 1, 1, :hash},
|
||||||
|
"HEXISTS" => {3, [:readonly, :fast], 1, 1, 1, :hash},
|
||||||
|
"HINCRBY" => {4, [:write, :denyoom, :fast], 1, 1, 1, :hash},
|
||||||
|
"HSCAN" => {-3, [:readonly], 1, 1, 1, :hash},
|
||||||
|
"SADD" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :set},
|
||||||
|
"SREM" => {-3, [:write, :fast], 1, 1, 1, :set},
|
||||||
|
"SMEMBERS" => {2, [:readonly], 1, 1, 1, :set},
|
||||||
|
"SISMEMBER" => {3, [:readonly, :fast], 1, 1, 1, :set},
|
||||||
|
"SMISMEMBER" => {-3, [:readonly, :fast], 1, 1, 1, :set},
|
||||||
|
"SCARD" => {2, [:readonly, :fast], 1, 1, 1, :set},
|
||||||
|
"SPOP" => {-2, [:write, :fast], 1, 1, 1, :set},
|
||||||
|
"SRANDMEMBER" => {-2, [:readonly], 1, 1, 1, :set},
|
||||||
|
"SMOVE" => {4, [:write, :fast], 1, 2, 1, :set},
|
||||||
|
"SINTER" => {-2, [:readonly], 1, -1, 1, :set},
|
||||||
|
"SUNION" => {-2, [:readonly], 1, -1, 1, :set},
|
||||||
|
"SDIFF" => {-2, [:readonly], 1, -1, 1, :set},
|
||||||
|
"SINTERCARD" => {-3, [:readonly, :movablekeys], 0, 0, 0, :set},
|
||||||
|
"SINTERSTORE" => {-3, [:write, :denyoom], 1, -1, 1, :set},
|
||||||
|
"SUNIONSTORE" => {-3, [:write, :denyoom], 1, -1, 1, :set},
|
||||||
|
"SDIFFSTORE" => {-3, [:write, :denyoom], 1, -1, 1, :set},
|
||||||
|
"SSCAN" => {-3, [:readonly], 1, 1, 1, :set},
|
||||||
|
"ZADD" => {-4, [:write, :denyoom, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZINCRBY" => {4, [:write, :denyoom, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZREM" => {-3, [:write, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZSCORE" => {3, [:readonly, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZMSCORE" => {-3, [:readonly, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZRANK" => {3, [:readonly, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZREVRANK" => {3, [:readonly, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZCARD" => {2, [:readonly, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZCOUNT" => {4, [:readonly, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZRANGE" => {-4, [:readonly], 1, 1, 1, :zset},
|
||||||
|
"ZREVRANGE" => {-4, [:readonly], 1, 1, 1, :zset},
|
||||||
|
"ZRANGEBYSCORE" => {-4, [:readonly], 1, 1, 1, :zset},
|
||||||
|
"ZPOPMIN" => {-2, [:write, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZPOPMAX" => {-2, [:write, :fast], 1, 1, 1, :zset},
|
||||||
|
"ZSCAN" => {-3, [:readonly], 1, 1, 1, :zset},
|
||||||
|
"XADD" => {-5, [:write, :denyoom, :fast], 1, 1, 1, :stream},
|
||||||
|
"XLEN" => {2, [:readonly, :fast], 1, 1, 1, :stream},
|
||||||
|
"XRANGE" => {-4, [:readonly], 1, 1, 1, :stream},
|
||||||
|
"XREVRANGE" => {-4, [:readonly], 1, 1, 1, :stream},
|
||||||
|
"XDEL" => {-3, [:write, :fast], 1, 1, 1, :stream},
|
||||||
|
"XTRIM" => {-4, [:write], 1, 1, 1, :stream},
|
||||||
|
"XSETID" => {-3, [:write, :denyoom, :fast], 1, 1, 1, :stream},
|
||||||
|
"XACK" => {-4, [:write, :fast], 1, 1, 1, :stream},
|
||||||
|
"XPENDING" => {-3, [:readonly], 1, 1, 1, :stream},
|
||||||
|
"XCLAIM" => {-6, [:write, :fast], 1, 1, 1, :stream},
|
||||||
|
"XAUTOCLAIM" => {-7, [:write, :fast], 1, 1, 1, :stream},
|
||||||
|
"XREAD" => {-4, [:readonly, :blocking, :movablekeys], 0, 0, 0, :stream},
|
||||||
|
"XREADGROUP" => {-7, [:write, :blocking, :movablekeys], 0, 0, 0, :stream},
|
||||||
|
"XINFO" => {-2, [:readonly], 0, 0, 0, :stream},
|
||||||
|
"XGROUP" => {-2, [:write, :denyoom], 0, 0, 0, :stream},
|
||||||
|
"SETBIT" => {4, [:write, :denyoom], 1, 1, 1, :bitmap},
|
||||||
|
"GETBIT" => {3, [:readonly, :fast], 1, 1, 1, :bitmap},
|
||||||
|
"BITCOUNT" => {-2, [:readonly], 1, 1, 1, :bitmap},
|
||||||
|
"BITPOS" => {-3, [:readonly], 1, 1, 1, :bitmap},
|
||||||
|
"BITFIELD" => {-2, [:write, :denyoom], 1, 1, 1, :bitmap},
|
||||||
|
"BITOP" => {-4, [:write, :denyoom], 2, -1, 1, :bitmap},
|
||||||
|
"PFADD" => {-2, [:write, :denyoom, :fast], 1, 1, 1, :hyperloglog},
|
||||||
|
"PFCOUNT" => {-2, [:readonly], 1, -1, 1, :hyperloglog},
|
||||||
|
"PFMERGE" => {-2, [:write, :denyoom], 1, -1, 1, :hyperloglog},
|
||||||
|
"SUBSCRIBE" => {-2, [:pubsub, :noscript, :loading, :stale, :fast], 0, 0, 0, :pubsub},
|
||||||
|
"UNSUBSCRIBE" => {-1, [:pubsub, :noscript, :loading, :stale, :fast], 0, 0, 0, :pubsub},
|
||||||
|
"PUBLISH" => {3, [:pubsub, :loading, :stale, :fast], 0, 0, 0, :pubsub},
|
||||||
|
"MULTI" => {1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :transactions},
|
||||||
|
"EXEC" => {1, [:noscript, :loading, :stale], 0, 0, 0, :transactions},
|
||||||
|
"DISCARD" => {1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :transactions},
|
||||||
|
"WATCH" => {-2, [:noscript, :loading, :stale, :fast], 1, -1, 1, :transactions},
|
||||||
|
"UNWATCH" => {1, [:noscript, :loading, :stale, :fast], 0, 0, 0, :transactions}
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyRules %{
|
||||||
|
"XREAD" => :streams,
|
||||||
|
"XREADGROUP" => :streams,
|
||||||
|
"SINTERCARD" => {:numkeys, 1},
|
||||||
|
"BLMPOP" => {:numkeys, 2},
|
||||||
|
"OBJECT" => {:subcommand, ~w(ENCODING REFCOUNT FREQ IDLETIME)},
|
||||||
|
"MEMORY" => {:subcommand, ~w(USAGE)},
|
||||||
|
"XINFO" => {:subcommand, ~w(STREAM GROUPS CONSUMERS)},
|
||||||
|
"XGROUP" => {:subcommand, ~w(CREATE SETID DESTROY CREATECONSUMER DELCONSUMER)}
|
||||||
|
}
|
||||||
|
|
||||||
|
def tableNames, do: Map.keys(@commands)
|
||||||
|
|
||||||
|
def names, do: Excache.Command.known_commands() |> Enum.uniq() |> Enum.sort()
|
||||||
|
|
||||||
|
def known?(name), do: Map.has_key?(@commands, name)
|
||||||
|
|
||||||
|
def count, do: length(names())
|
||||||
|
|
||||||
|
def infoTerm(name) do
|
||||||
|
case Map.fetch(@commands, name) do
|
||||||
|
{:ok, {arity, flags, firstKey, lastKey, step, _group}} ->
|
||||||
|
{:array,
|
||||||
|
[
|
||||||
|
{:bulk, String.downcase(name)},
|
||||||
|
{:integer, arity},
|
||||||
|
{:array, Enum.map(flags, fn flag -> {:simple, Atom.to_string(flag)} end)},
|
||||||
|
{:integer, firstKey},
|
||||||
|
{:integer, lastKey},
|
||||||
|
{:integer, step}
|
||||||
|
]}
|
||||||
|
|
||||||
|
:error ->
|
||||||
|
{:array, nil}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def docsTerm(name) do
|
||||||
|
{arity, _flags, _firstKey, _lastKey, _step, group} = Map.fetch!(@commands, name)
|
||||||
|
|
||||||
|
{:array,
|
||||||
|
[
|
||||||
|
{:bulk, "group"},
|
||||||
|
{:bulk, Atom.to_string(group)},
|
||||||
|
{:bulk, "arity"},
|
||||||
|
{:integer, arity}
|
||||||
|
]}
|
||||||
|
end
|
||||||
|
|
||||||
|
def arityOk?(name, argc) do
|
||||||
|
{arity, _flags, _firstKey, _lastKey, _step, _group} = Map.fetch!(@commands, name)
|
||||||
|
if arity < 0, do: argc >= -arity, else: argc == arity
|
||||||
|
end
|
||||||
|
|
||||||
|
def keysFor(name, args) do
|
||||||
|
case Map.get(@keyRules, name) do
|
||||||
|
nil -> rangeKeys(name, args)
|
||||||
|
rule -> ruleKeys(rule, args)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp rangeKeys(name, args) do
|
||||||
|
{_arity, _flags, firstKey, lastKey, step, _group} = Map.fetch!(@commands, name)
|
||||||
|
|
||||||
|
if firstKey == 0 do
|
||||||
|
{:ok, []}
|
||||||
|
else
|
||||||
|
count = length(args)
|
||||||
|
from = firstKey - 1
|
||||||
|
to = if lastKey < 0, do: count + lastKey, else: lastKey - 1
|
||||||
|
|
||||||
|
if to < from do
|
||||||
|
{:ok, []}
|
||||||
|
else
|
||||||
|
{:ok, args |> Enum.slice(from..to) |> Enum.take_every(step)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp ruleKeys(:streams, args) do
|
||||||
|
case Enum.find_index(args, fn arg -> String.upcase(arg) == "STREAMS" end) do
|
||||||
|
nil ->
|
||||||
|
:error
|
||||||
|
|
||||||
|
index ->
|
||||||
|
tail = Enum.drop(args, index + 1)
|
||||||
|
half = div(length(tail), 2)
|
||||||
|
if half == 0 or rem(length(tail), 2) == 1, do: :error, else: {:ok, Enum.take(tail, half)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp ruleKeys({:numkeys, position}, args) do
|
||||||
|
with {numkeys, ""} when numkeys > 0 <- Integer.parse(Enum.at(args, position - 1, "")),
|
||||||
|
keys when length(keys) == numkeys <- args |> Enum.drop(position) |> Enum.take(numkeys) do
|
||||||
|
{:ok, keys}
|
||||||
|
else
|
||||||
|
_invalid -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp ruleKeys({:subcommand, takesKey}, [subcommand, key | _rest]) do
|
||||||
|
if String.upcase(subcommand) in takesKey, do: {:ok, [key]}, else: {:ok, []}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp ruleKeys({:subcommand, _takesKey}, _args), do: {:ok, []}
|
||||||
|
end
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
defmodule Excache.Command.Introspection.Client do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Clients
|
||||||
|
alias Excache.Command.Reply
|
||||||
|
|
||||||
|
@help [
|
||||||
|
"CLIENT <subcommand> [<arg> [value] [opt] ...]. Subcommands are:",
|
||||||
|
"ID",
|
||||||
|
" Return the client id of the current connection.",
|
||||||
|
"INFO",
|
||||||
|
" Return information about the current connection.",
|
||||||
|
"GETNAME",
|
||||||
|
" Return the name of the current connection.",
|
||||||
|
"SETNAME <name>",
|
||||||
|
" Assign the name <name> to the current connection.",
|
||||||
|
"LIST [ID <id> [<id> ...]] [TYPE normal|master|replica|pubsub]",
|
||||||
|
" Return one line of information per connection.",
|
||||||
|
"KILL <ip:port>",
|
||||||
|
" Close the connection reachable at <ip:port>.",
|
||||||
|
"KILL <filter> <value> [<filter> <value> ...]",
|
||||||
|
" Close connections matching ID, ADDR, LADDR, TYPE, MAXAGE or SKIPME.",
|
||||||
|
"HELP",
|
||||||
|
" Print this help."
|
||||||
|
]
|
||||||
|
|
||||||
|
@killFilters ~w(ID ADDR LADDR TYPE MAXAGE SKIPME)
|
||||||
|
@clientTypes ~w(normal master replica pubsub)
|
||||||
|
|
||||||
|
def dispatch("ID", [], state),
|
||||||
|
do: withClient(state, fn client -> {:integer, client.id} end)
|
||||||
|
|
||||||
|
def dispatch("INFO", [], state),
|
||||||
|
do: withClient(state, fn client -> {:bulk, line(client, now())} end)
|
||||||
|
|
||||||
|
def dispatch("GETNAME", [], state),
|
||||||
|
do: withClient(state, fn client -> {:bulk, client.name} end)
|
||||||
|
|
||||||
|
def dispatch("SETNAME", [name], state) do
|
||||||
|
if validName?(name) do
|
||||||
|
withClient(state, fn client ->
|
||||||
|
:ok = Clients.setName(client.pid, name)
|
||||||
|
{:simple, "OK"}
|
||||||
|
end)
|
||||||
|
else
|
||||||
|
{[{:error, "ERR Client names cannot contain spaces, newlines or special characters."}],
|
||||||
|
state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("LIST", args, state) do
|
||||||
|
case listFilter(args) do
|
||||||
|
{:ok, matcher} ->
|
||||||
|
moment = now()
|
||||||
|
|
||||||
|
text =
|
||||||
|
Clients.snapshot().clients
|
||||||
|
|> Enum.filter(matcher)
|
||||||
|
|> Enum.map_join("", fn client -> line(client, moment) <> "\n" end)
|
||||||
|
|
||||||
|
{[{:bulk, text}], state}
|
||||||
|
|
||||||
|
:error ->
|
||||||
|
{[Reply.syntax()], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("KILL", [address], state) do
|
||||||
|
killed = Clients.kill(fn client -> client.addr == address end, self())
|
||||||
|
|
||||||
|
if killed > 0 do
|
||||||
|
{[{:simple, "OK"}], state}
|
||||||
|
else
|
||||||
|
{[{:error, "ERR No such client address in the client list"}], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("KILL", [_, _ | _] = args, state) do
|
||||||
|
case killFilter(args) do
|
||||||
|
{:ok, matcher} -> {[{:integer, Clients.kill(matcher, self())}], state}
|
||||||
|
:error -> {[Reply.syntax()], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("HELP", [], state),
|
||||||
|
do: {[{:array, Enum.map(@help, fn text -> {:simple, text} end)}], state}
|
||||||
|
|
||||||
|
def dispatch(subcommand, _args, state) do
|
||||||
|
{[
|
||||||
|
{:error,
|
||||||
|
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
|
||||||
|
"Try CLIENT HELP."}
|
||||||
|
], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp line(client, moment) do
|
||||||
|
Enum.join(
|
||||||
|
[
|
||||||
|
"id=#{client.id}",
|
||||||
|
"addr=#{client.addr}",
|
||||||
|
"laddr=#{client.laddr}",
|
||||||
|
"name=#{client.name}",
|
||||||
|
"age=#{elapsed(moment, client.connectedAt)}",
|
||||||
|
"idle=#{elapsed(moment, client.lastActiveAt)}",
|
||||||
|
"flags=N",
|
||||||
|
"db=0",
|
||||||
|
"sub=#{Clients.subscriptionCount(client.pid)}",
|
||||||
|
"psub=0",
|
||||||
|
"tot-net-in=#{client.netIn}",
|
||||||
|
"tot-net-out=#{client.netOut}"
|
||||||
|
],
|
||||||
|
" "
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp withClient(state, build) do
|
||||||
|
case Clients.find(self()) do
|
||||||
|
nil -> {[{:error, "ERR CLIENT is only available on a server connection"}], state}
|
||||||
|
client -> {[build.(client)], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp listFilter([]), do: {:ok, fn _client -> true end}
|
||||||
|
|
||||||
|
defp listFilter([keyword | [_ | _] = rest]) do
|
||||||
|
case String.upcase(keyword) do
|
||||||
|
"ID" -> idFilter(rest)
|
||||||
|
"TYPE" -> typeFilter(rest)
|
||||||
|
_other -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp listFilter(_args), do: :error
|
||||||
|
|
||||||
|
defp idFilter(raw) do
|
||||||
|
ids = Enum.map(raw, &parseId/1)
|
||||||
|
|
||||||
|
if :error in ids do
|
||||||
|
:error
|
||||||
|
else
|
||||||
|
wanted = MapSet.new(ids)
|
||||||
|
{:ok, fn client -> MapSet.member?(wanted, client.id) end}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp typeFilter([type]) do
|
||||||
|
case String.downcase(type) do
|
||||||
|
"normal" -> {:ok, fn client -> Clients.subscriptionCount(client.pid) == 0 end}
|
||||||
|
"pubsub" -> {:ok, fn client -> Clients.subscriptionCount(client.pid) > 0 end}
|
||||||
|
"master" -> {:ok, fn _client -> false end}
|
||||||
|
"replica" -> {:ok, fn _client -> false end}
|
||||||
|
_other -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp typeFilter(_raw), do: :error
|
||||||
|
|
||||||
|
defp killFilter(args) do
|
||||||
|
case parseFilters(args, %{skipme: true}) do
|
||||||
|
{:ok, filters} -> {:ok, buildMatcher(filters)}
|
||||||
|
:error -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseFilters([], filters), do: {:ok, filters}
|
||||||
|
|
||||||
|
defp parseFilters([keyword, value | rest], filters) do
|
||||||
|
upcased = String.upcase(keyword)
|
||||||
|
|
||||||
|
if upcased in @killFilters do
|
||||||
|
case addFilter(upcased, value, filters) do
|
||||||
|
{:ok, updated} -> parseFilters(rest, updated)
|
||||||
|
:error -> :error
|
||||||
|
end
|
||||||
|
else
|
||||||
|
:error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseFilters(_args, _filters), do: :error
|
||||||
|
|
||||||
|
defp addFilter("ID", value, filters) do
|
||||||
|
case parseId(value) do
|
||||||
|
:error -> :error
|
||||||
|
id -> {:ok, Map.put(filters, :id, id)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp addFilter("ADDR", value, filters), do: {:ok, Map.put(filters, :addr, value)}
|
||||||
|
defp addFilter("LADDR", value, filters), do: {:ok, Map.put(filters, :laddr, value)}
|
||||||
|
|
||||||
|
defp addFilter("TYPE", value, filters) do
|
||||||
|
lowered = String.downcase(value)
|
||||||
|
if lowered in @clientTypes, do: {:ok, Map.put(filters, :type, lowered)}, else: :error
|
||||||
|
end
|
||||||
|
|
||||||
|
defp addFilter("MAXAGE", value, filters) do
|
||||||
|
case Integer.parse(value) do
|
||||||
|
{seconds, ""} when seconds >= 0 -> {:ok, Map.put(filters, :maxage, seconds)}
|
||||||
|
_invalid -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp addFilter("SKIPME", value, filters) do
|
||||||
|
case String.downcase(value) do
|
||||||
|
"yes" -> {:ok, Map.put(filters, :skipme, true)}
|
||||||
|
"no" -> {:ok, Map.put(filters, :skipme, false)}
|
||||||
|
_other -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp buildMatcher(filters) do
|
||||||
|
caller = self()
|
||||||
|
moment = now()
|
||||||
|
|
||||||
|
fn client ->
|
||||||
|
matchesId?(filters, client) and matchesAddr?(filters, client) and
|
||||||
|
matchesLaddr?(filters, client) and matchesType?(filters, client) and
|
||||||
|
matchesAge?(filters, client, moment) and matchesSkipme?(filters, client, caller)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp matchesId?(%{id: id}, client), do: client.id == id
|
||||||
|
defp matchesId?(_filters, _client), do: true
|
||||||
|
|
||||||
|
defp matchesAddr?(%{addr: addr}, client), do: client.addr == addr
|
||||||
|
defp matchesAddr?(_filters, _client), do: true
|
||||||
|
|
||||||
|
defp matchesLaddr?(%{laddr: laddr}, client), do: client.laddr == laddr
|
||||||
|
defp matchesLaddr?(_filters, _client), do: true
|
||||||
|
|
||||||
|
defp matchesType?(%{type: "normal"}, client), do: Clients.subscriptionCount(client.pid) == 0
|
||||||
|
defp matchesType?(%{type: "pubsub"}, client), do: Clients.subscriptionCount(client.pid) > 0
|
||||||
|
defp matchesType?(%{type: _other}, _client), do: false
|
||||||
|
defp matchesType?(_filters, _client), do: true
|
||||||
|
|
||||||
|
defp matchesAge?(%{maxage: maxage}, client, moment),
|
||||||
|
do: elapsed(moment, client.connectedAt) >= maxage
|
||||||
|
|
||||||
|
defp matchesAge?(_filters, _client, _moment), do: true
|
||||||
|
|
||||||
|
defp matchesSkipme?(%{skipme: true}, client, caller), do: client.pid != caller
|
||||||
|
defp matchesSkipme?(_filters, _client, _caller), do: true
|
||||||
|
|
||||||
|
defp parseId(raw) do
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{id, ""} when id > 0 -> id
|
||||||
|
_invalid -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp validName?(name), do: not String.contains?(name, [" ", "\n", "\r"])
|
||||||
|
|
||||||
|
defp elapsed(moment, since), do: div(moment - since, 1000)
|
||||||
|
|
||||||
|
defp now, do: System.system_time(:millisecond)
|
||||||
|
end
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
defmodule Excache.Command.Introspection.Info do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Clients
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
alias Excache.Memory
|
||||||
|
alias Excache.Persistence
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@redisCompat "7.2.0"
|
||||||
|
|
||||||
|
@sections %{
|
||||||
|
"server" => :server,
|
||||||
|
"clients" => :clients,
|
||||||
|
"memory" => :memory,
|
||||||
|
"persistence" => :persistence,
|
||||||
|
"stats" => :stats,
|
||||||
|
"replication" => :replication,
|
||||||
|
"keyspace" => :keyspace,
|
||||||
|
"cpu" => :cpu
|
||||||
|
}
|
||||||
|
|
||||||
|
@default [:server, :clients, :memory, :persistence, :stats, :replication, :keyspace]
|
||||||
|
@everything @default ++ [:cpu]
|
||||||
|
|
||||||
|
@titles %{
|
||||||
|
server: "Server",
|
||||||
|
clients: "Clients",
|
||||||
|
memory: "Memory",
|
||||||
|
persistence: "Persistence",
|
||||||
|
stats: "Stats",
|
||||||
|
replication: "Replication",
|
||||||
|
keyspace: "Keyspace",
|
||||||
|
cpu: "Cpu"
|
||||||
|
}
|
||||||
|
|
||||||
|
@clientSections [:server, :clients, :stats]
|
||||||
|
@keyspaceSections [:memory, :keyspace]
|
||||||
|
|
||||||
|
def render(args), do: args |> requested() |> emit()
|
||||||
|
|
||||||
|
defp requested([]), do: @default
|
||||||
|
|
||||||
|
defp requested(args) do
|
||||||
|
args
|
||||||
|
|> Enum.flat_map(fn arg ->
|
||||||
|
case String.downcase(arg) do
|
||||||
|
"all" -> @everything
|
||||||
|
"everything" -> @everything
|
||||||
|
"default" -> @default
|
||||||
|
name -> @sections |> Map.get(name) |> List.wrap()
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> Enum.uniq()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp emit([]), do: ""
|
||||||
|
|
||||||
|
defp emit(sections) do
|
||||||
|
context = context(sections)
|
||||||
|
|
||||||
|
sections
|
||||||
|
|> Enum.map(fn section -> block(Map.fetch!(@titles, section), lines(section, context)) end)
|
||||||
|
|> IO.iodata_to_binary()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp block(title, fields) do
|
||||||
|
[
|
||||||
|
"# ",
|
||||||
|
title,
|
||||||
|
"\r\n",
|
||||||
|
Enum.map(fields, fn {key, value} -> [key, ":", text(value), "\r\n"] end),
|
||||||
|
"\r\n"
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp text(value) when is_binary(value), do: value
|
||||||
|
defp text(value) when is_integer(value), do: Integer.to_string(value)
|
||||||
|
defp text(value) when is_atom(value), do: Atom.to_string(value)
|
||||||
|
defp text(value) when is_list(value), do: List.to_string(value)
|
||||||
|
|
||||||
|
defp context(sections) do
|
||||||
|
%{}
|
||||||
|
|> withClients(Enum.any?(sections, &(&1 in @clientSections)))
|
||||||
|
|> withKeyspace(Enum.any?(sections, &(&1 in @keyspaceSections)))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp withClients(context, false), do: context
|
||||||
|
defp withClients(context, true), do: Map.put(context, :clients, Clients.snapshot())
|
||||||
|
|
||||||
|
defp withKeyspace(context, false), do: context
|
||||||
|
|
||||||
|
defp withKeyspace(context, true) do
|
||||||
|
entries = Store.dump()
|
||||||
|
now = System.system_time(:millisecond)
|
||||||
|
remaining = for {_key, _kind, _value, at} <- entries, at != nil and at > now, do: at - now
|
||||||
|
|
||||||
|
Map.merge(context, %{
|
||||||
|
memory: Memory.stats(entries),
|
||||||
|
volatile: length(remaining),
|
||||||
|
avgTtl: averageTtl(remaining)
|
||||||
|
})
|
||||||
|
end
|
||||||
|
|
||||||
|
defp averageTtl([]), do: 0
|
||||||
|
defp averageTtl(remaining), do: div(Enum.sum(remaining), length(remaining))
|
||||||
|
|
||||||
|
defp lines(:server, context) do
|
||||||
|
snapshot = context.clients
|
||||||
|
uptime = div(System.system_time(:millisecond) - snapshot.startedAt, 1000)
|
||||||
|
|
||||||
|
[
|
||||||
|
{"excache_version", version()},
|
||||||
|
{"redis_version", @redisCompat},
|
||||||
|
{"redis_mode", "standalone"},
|
||||||
|
{"os", :erlang.system_info(:system_architecture)},
|
||||||
|
{"arch_bits", :erlang.system_info(:wordsize) * 8},
|
||||||
|
{"process_id", :os.getpid()},
|
||||||
|
{"run_id", snapshot.runId},
|
||||||
|
{"tcp_port", tcpPort()},
|
||||||
|
{"uptime_in_seconds", uptime},
|
||||||
|
{"uptime_in_days", div(uptime, 86_400)},
|
||||||
|
{"otp_release", :erlang.system_info(:otp_release)},
|
||||||
|
{"erts_version", :erlang.system_info(:version)},
|
||||||
|
{"elixir_version", System.version()}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:clients, context) do
|
||||||
|
clients = context.clients.clients
|
||||||
|
|
||||||
|
[
|
||||||
|
{"connected_clients", length(clients)},
|
||||||
|
{"blocked_clients", Enum.count(clients, fn client -> Clients.blocked?(client.pid) end)},
|
||||||
|
{"pubsub_clients",
|
||||||
|
Enum.count(clients, fn client -> Clients.subscriptionCount(client.pid) > 0 end)}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:memory, context) do
|
||||||
|
stats = context.memory
|
||||||
|
limit = Config.maxmemory()
|
||||||
|
|
||||||
|
[
|
||||||
|
{"used_memory", stats.usedMemory},
|
||||||
|
{"used_memory_human", humanBytes(stats.usedMemory)},
|
||||||
|
{"used_memory_dataset", stats.datasetBytes},
|
||||||
|
{"used_memory_overhead", stats.overheadBytes},
|
||||||
|
{"used_memory_beam", :erlang.memory(:total)},
|
||||||
|
{"used_memory_beam_processes", :erlang.memory(:processes)},
|
||||||
|
{"used_memory_beam_binary", :erlang.memory(:binary)},
|
||||||
|
{"used_memory_beam_ets", :erlang.memory(:ets)},
|
||||||
|
{"maxmemory", limit},
|
||||||
|
{"maxmemory_human", humanBytes(limit)},
|
||||||
|
{"maxmemory_policy", Eviction.policyName(Config.policy())},
|
||||||
|
{"maxmemory_samples", Config.maxmemorySamples()},
|
||||||
|
{"mem_allocator", "erts"}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:persistence, _context) do
|
||||||
|
snapshotPath = Persistence.snapshot_path()
|
||||||
|
|
||||||
|
[
|
||||||
|
{"loading", 0},
|
||||||
|
{"rdb_last_save_time", fileMtime(snapshotPath)},
|
||||||
|
{"rdb_snapshot_size", fileSize(snapshotPath)},
|
||||||
|
{"aof_enabled", 1},
|
||||||
|
{"aof_rewrite_in_progress", 0},
|
||||||
|
{"aof_fsync_policy", Aof.fsync_policy()},
|
||||||
|
{"aof_current_size", fileSize(Aof.path())}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:stats, context) do
|
||||||
|
snapshot = context.clients
|
||||||
|
runner = Runner.stats()
|
||||||
|
|
||||||
|
[
|
||||||
|
{"total_connections_received", snapshot.connectionsReceived},
|
||||||
|
{"total_net_input_bytes", snapshot.netIn},
|
||||||
|
{"total_net_output_bytes", snapshot.netOut},
|
||||||
|
{"evicted_keys", runner.evictedKeys},
|
||||||
|
{"oom_rejections", runner.oomRejections},
|
||||||
|
{"pubsub_channels", Clients.subscribedChannels()},
|
||||||
|
{"pubsub_patterns", 0}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:replication, _context) do
|
||||||
|
[
|
||||||
|
{"role", "master"},
|
||||||
|
{"connected_slaves", 0},
|
||||||
|
{"master_failover_state", "no-failover"},
|
||||||
|
{"master_repl_offset", 0},
|
||||||
|
{"second_repl_offset", -1},
|
||||||
|
{"repl_backlog_active", 0},
|
||||||
|
{"repl_backlog_size", 0},
|
||||||
|
{"repl_backlog_first_byte_offset", 0},
|
||||||
|
{"repl_backlog_histlen", 0}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:keyspace, context) do
|
||||||
|
case Store.dbsize() do
|
||||||
|
0 ->
|
||||||
|
[]
|
||||||
|
|
||||||
|
keys ->
|
||||||
|
[{"db0", "keys=#{keys},expires=#{context.volatile},avg_ttl=#{context.avgTtl}"}]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lines(:cpu, _context) do
|
||||||
|
{runtimeMillis, _sinceLast} = :erlang.statistics(:runtime)
|
||||||
|
{wallMillis, _sinceLast} = :erlang.statistics(:wall_clock)
|
||||||
|
|
||||||
|
[
|
||||||
|
{"used_cpu_total", seconds(runtimeMillis)},
|
||||||
|
{"wall_clock_seconds", seconds(wallMillis)},
|
||||||
|
{"schedulers", :erlang.system_info(:schedulers)},
|
||||||
|
{"schedulers_online", :erlang.system_info(:schedulers_online)},
|
||||||
|
{"run_queue", :erlang.statistics(:total_run_queue_lengths)}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp version, do: :excache |> Application.spec(:vsn) |> List.to_string()
|
||||||
|
|
||||||
|
defp tcpPort do
|
||||||
|
{:ok, port} = Config.get("port")
|
||||||
|
port || 0
|
||||||
|
end
|
||||||
|
|
||||||
|
defp seconds(millis), do: :erlang.float_to_binary(millis / 1000, decimals: 2)
|
||||||
|
|
||||||
|
defp fileSize(path) do
|
||||||
|
case File.stat(path) do
|
||||||
|
{:ok, stat} -> stat.size
|
||||||
|
{:error, _reason} -> 0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fileMtime(path) do
|
||||||
|
case File.stat(path, time: :posix) do
|
||||||
|
{:ok, stat} -> stat.mtime
|
||||||
|
{:error, _reason} -> 0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp humanBytes(bytes) when bytes < 1024, do: "#{bytes}B"
|
||||||
|
defp humanBytes(bytes) when bytes < 1_048_576, do: scaled(bytes, 1024, "K")
|
||||||
|
defp humanBytes(bytes) when bytes < 1_073_741_824, do: scaled(bytes, 1_048_576, "M")
|
||||||
|
defp humanBytes(bytes), do: scaled(bytes, 1_073_741_824, "G")
|
||||||
|
|
||||||
|
defp scaled(bytes, divisor, suffix),
|
||||||
|
do: :erlang.float_to_binary(bytes / divisor, decimals: 2) <> suffix
|
||||||
|
end
|
||||||
@@ -0,0 +1,504 @@
|
|||||||
|
defmodule Excache.Command.Keyspace do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.{SkipList, Store}
|
||||||
|
alias Excache.Command.Keyspace.Scan
|
||||||
|
alias Excache.Command.Reply
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
|
||||||
|
@scanFamily ~w(SCAN HSCAN SSCAN ZSCAN)
|
||||||
|
@sharedRefcount 2_147_483_647
|
||||||
|
@listpackEntries 128
|
||||||
|
@listpackValue 64
|
||||||
|
@intsetEntries 512
|
||||||
|
|
||||||
|
def dispatch(name, args, state) when name in @scanFamily,
|
||||||
|
do: Scan.dispatch(name, args, state)
|
||||||
|
|
||||||
|
def dispatch("OBJECT", [subcommand | rest], state),
|
||||||
|
do: object(String.upcase(subcommand), rest, state)
|
||||||
|
|
||||||
|
def dispatch("LPOS", [key, element | rest], state) do
|
||||||
|
case parseLposOptions(rest, %{rank: 1, count: nil, maxlen: 0}) do
|
||||||
|
{:ok, options} -> lpos(key, element, options, state)
|
||||||
|
{:reply, term} -> reply(term, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("STRLEN", [key], state), do: reply(integerReply(Store.strlen(key)), state)
|
||||||
|
|
||||||
|
def dispatch("SETRANGE", [key, offsetRaw, value], state) do
|
||||||
|
case Integer.parse(offsetRaw) do
|
||||||
|
{offset, ""} when offset >= 0 -> reply(setrange(key, offset, value), state)
|
||||||
|
{_negative, ""} -> reply({:error, "ERR offset is out of range"}, state)
|
||||||
|
_ -> reply(Reply.not_integer(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("GETRANGE", [key, startRaw, stopRaw], state) do
|
||||||
|
Reply.with_two_ints(startRaw, stopRaw, state, fn start, stop ->
|
||||||
|
bulkReply(Store.getrange(key, start, stop))
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("SETNX", [key, value], state),
|
||||||
|
do: reply({:integer, Store.setnx(key, value)}, state)
|
||||||
|
|
||||||
|
def dispatch("GETDEL", [key], state), do: reply(bulkReply(Store.getdel(key)), state)
|
||||||
|
|
||||||
|
def dispatch("INCRBYFLOAT", [key, deltaRaw], state) do
|
||||||
|
case Reply.parse_score(deltaRaw) do
|
||||||
|
{:ok, delta} -> reply(bulkReply(Store.incrbyfloat(key, delta)), state)
|
||||||
|
:error -> reply(Reply.not_float(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("HSETNX", [key, field, value], state),
|
||||||
|
do: reply(integerReply(Store.hsetnx(key, field, value)), state)
|
||||||
|
|
||||||
|
def dispatch("HMGET", [key | [_ | _] = fields], state) do
|
||||||
|
withType(key, "hash", state, fn ->
|
||||||
|
{:array, Reply.bulks_nullable(Store.hmget(key, fields))}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("SMISMEMBER", [key | [_ | _] = members], state) do
|
||||||
|
withType(key, "set", state, fn ->
|
||||||
|
{:array, Enum.map(Store.smismember(key, members), &{:integer, &1})}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("SINTERCARD", [numkeysRaw | [_ | _] = rest], state) do
|
||||||
|
case parseIntercard(numkeysRaw, rest) do
|
||||||
|
{:ok, keys, limit} -> reply(integerReply(Store.sintercard(keys, limit)), state)
|
||||||
|
{:reply, term} -> reply(term, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("SPOP", [key], state), do: reply(singleReply(Store.spop(key, nil)), state)
|
||||||
|
|
||||||
|
def dispatch("SPOP", [key, countRaw], state) do
|
||||||
|
case Integer.parse(countRaw) do
|
||||||
|
{count, ""} when count >= 0 -> reply(arrayReply(Store.spop(key, count)), state)
|
||||||
|
{_negative, ""} -> reply({:error, "ERR value is out of range, must be positive"}, state)
|
||||||
|
_ -> reply(Reply.not_integer(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("SRANDMEMBER", [key], state),
|
||||||
|
do: reply(singleReply(Store.srandmember(key, nil)), state)
|
||||||
|
|
||||||
|
def dispatch("SRANDMEMBER", [key, countRaw], state) do
|
||||||
|
Reply.with_int(countRaw, state, fn count -> arrayReply(Store.srandmember(key, count)) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("SMOVE", [source, dest, member], state) do
|
||||||
|
withType(dest, "set", state, fn -> integerReply(Store.smove(source, dest, member)) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("LREM", [key, countRaw, value], state) do
|
||||||
|
Reply.with_int(countRaw, state, fn count -> integerReply(Store.lrem(key, count, value)) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("LSET", [key, indexRaw, value], state) do
|
||||||
|
Reply.with_int(indexRaw, state, fn index -> statusReply(Store.lset(key, index, value)) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("LTRIM", [key, startRaw, stopRaw], state) do
|
||||||
|
Reply.with_two_ints(startRaw, stopRaw, state, fn start, stop ->
|
||||||
|
statusReply(Store.ltrim(key, start, stop))
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("LINSERT", [key, positionRaw, pivot, value], state) do
|
||||||
|
case String.upcase(positionRaw) do
|
||||||
|
"BEFORE" -> reply(integerReply(Store.linsert(key, :before, pivot, value)), state)
|
||||||
|
"AFTER" -> reply(integerReply(Store.linsert(key, :after, pivot, value)), state)
|
||||||
|
_ -> reply(Reply.syntax(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("RPOPLPUSH", [source, dest], state) do
|
||||||
|
withType(dest, "list", state, fn -> bulkReply(Store.rpoplpush(source, dest)) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("ZCOUNT", [key, minRaw, maxRaw], state) do
|
||||||
|
case {Reply.parse_bound(minRaw), Reply.parse_bound(maxRaw)} do
|
||||||
|
{{:ok, minBound}, {:ok, maxBound}} ->
|
||||||
|
reply(integerReply(Store.zcount(key, minBound, maxBound)), state)
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
reply({:error, "ERR min or max is not a float"}, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("ZREVRANGE", [key, startRaw, stopRaw], state),
|
||||||
|
do: zrevrange(key, startRaw, stopRaw, false, state)
|
||||||
|
|
||||||
|
def dispatch("ZREVRANGE", [key, startRaw, stopRaw, flag], state) do
|
||||||
|
if String.upcase(flag) == "WITHSCORES" do
|
||||||
|
zrevrange(key, startRaw, stopRaw, true, state)
|
||||||
|
else
|
||||||
|
reply(Reply.syntax(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("ZREVRANK", [key, member], state),
|
||||||
|
do: reply(rankReply(Store.zrevrank(key, member)), state)
|
||||||
|
|
||||||
|
def dispatch("ZMSCORE", [key | [_ | _] = members], state) do
|
||||||
|
case Store.zmscore(key, members) do
|
||||||
|
{:ok, scores} -> reply({:array, Enum.map(scores, &scoreEntry/1)}, state)
|
||||||
|
{:error, reason} -> reply(errorTerm(reason), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("RENAME", [source, dest], state),
|
||||||
|
do: reply(statusReply(Store.rename(source, dest)), state)
|
||||||
|
|
||||||
|
def dispatch("RENAMENX", [source, dest], state),
|
||||||
|
do: reply(integerReply(Store.renamenx(source, dest)), state)
|
||||||
|
|
||||||
|
def dispatch("COPY", [source, dest | rest], state) do
|
||||||
|
case parseCopyOptions(rest, false) do
|
||||||
|
{:ok, _replace} when source == dest ->
|
||||||
|
reply({:error, "ERR source and destination objects are the same"}, state)
|
||||||
|
|
||||||
|
{:ok, replace} ->
|
||||||
|
reply(integerReply(Store.copy(source, dest, replace)), state)
|
||||||
|
|
||||||
|
{:reply, term} ->
|
||||||
|
reply(term, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("TOUCH", [_ | _] = keys, state), do: reply({:integer, Store.touch(keys)}, state)
|
||||||
|
def dispatch("UNLINK", [_ | _] = keys, state), do: reply({:integer, Store.del(keys)}, state)
|
||||||
|
|
||||||
|
def dispatch("EXPIREAT", [key, secondsRaw | rest], state) do
|
||||||
|
case Integer.parse(secondsRaw) do
|
||||||
|
{seconds, ""} ->
|
||||||
|
case parseExpireFlags(rest) do
|
||||||
|
{:ok, flags} -> reply({:integer, expireAt(key, seconds * 1000, flags)}, state)
|
||||||
|
{:reply, term} -> reply(term, state)
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
reply(Reply.not_integer(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("EXPIRETIME", [key], state),
|
||||||
|
do: reply({:integer, Store.expiretime(key, :seconds)}, state)
|
||||||
|
|
||||||
|
def dispatch("PEXPIRETIME", [key], state),
|
||||||
|
do: reply({:integer, Store.expiretime(key, :milliseconds)}, state)
|
||||||
|
|
||||||
|
def dispatch(_name, _args, _state), do: :unhandled
|
||||||
|
|
||||||
|
defp setrange(key, _offset, ""), do: integerReply(Store.strlen(key))
|
||||||
|
defp setrange(key, offset, value), do: integerReply(Store.setrange(key, offset, value))
|
||||||
|
|
||||||
|
defp lpos(key, element, %{count: nil} = options, state) do
|
||||||
|
case Store.lpos(key, element, options.rank, 1, options.maxlen) do
|
||||||
|
{:ok, []} -> reply({:bulk, nil}, state)
|
||||||
|
{:ok, [index]} -> reply({:integer, index}, state)
|
||||||
|
{:error, reason} -> reply(errorTerm(reason), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp lpos(key, element, options, state) do
|
||||||
|
case Store.lpos(key, element, options.rank, options.count, options.maxlen) do
|
||||||
|
{:ok, indexes} -> reply({:array, Enum.map(indexes, &{:integer, &1})}, state)
|
||||||
|
{:error, reason} -> reply(errorTerm(reason), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseLposOptions([], options), do: {:ok, options}
|
||||||
|
|
||||||
|
defp parseLposOptions([token | rest], options) do
|
||||||
|
case {String.upcase(token), rest} do
|
||||||
|
{"RANK", [raw | tail]} ->
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{0, ""} -> {:reply, {:error, rankZeroMessage()}}
|
||||||
|
{rank, ""} -> parseLposOptions(tail, %{options | rank: rank})
|
||||||
|
_ -> {:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
|
||||||
|
{"COUNT", [raw | tail]} ->
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{count, ""} when count >= 0 -> parseLposOptions(tail, %{options | count: count})
|
||||||
|
{_negative, ""} -> {:reply, {:error, "ERR COUNT can't be negative"}}
|
||||||
|
_ -> {:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
|
||||||
|
{"MAXLEN", [raw | tail]} ->
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{maxlen, ""} when maxlen >= 0 -> parseLposOptions(tail, %{options | maxlen: maxlen})
|
||||||
|
{_negative, ""} -> {:reply, {:error, "ERR MAXLEN can't be negative"}}
|
||||||
|
_ -> {:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp rankZeroMessage do
|
||||||
|
"ERR RANK can't be zero. Use 1 to start searching from the first matching element in the " <>
|
||||||
|
"head of the list or -1 searching backward from the tail of the list"
|
||||||
|
end
|
||||||
|
|
||||||
|
defp object("ENCODING", [key], state) do
|
||||||
|
withEntry(key, state, fn entry -> {:bulk, encodingName(entry)} end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp object("REFCOUNT", [key], state) do
|
||||||
|
withEntry(key, state, fn entry -> {:integer, refcount(entry)} end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp object("IDLETIME", [key], state) do
|
||||||
|
withEntry(key, state, fn _entry -> {:integer, idleSeconds(key)} end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp object("HELP", [], state) do
|
||||||
|
lines = [
|
||||||
|
"OBJECT <subcommand> <key>. Subcommands are:",
|
||||||
|
"ENCODING <key>",
|
||||||
|
" Return the kind of internal representation used to store the value of <key>.",
|
||||||
|
"IDLETIME <key>",
|
||||||
|
" Return the seconds since <key> was last accessed, or 0 when the current",
|
||||||
|
" maxmemory-policy keeps no access metadata for it.",
|
||||||
|
"REFCOUNT <key>",
|
||||||
|
" Return the number of references of the value associated with <key>.",
|
||||||
|
"HELP",
|
||||||
|
" Print this help."
|
||||||
|
]
|
||||||
|
|
||||||
|
reply({:array, Enum.map(lines, &{:simple, &1})}, state)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp object(subcommand, _rest, state) do
|
||||||
|
message =
|
||||||
|
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
|
||||||
|
"Try OBJECT HELP."
|
||||||
|
|
||||||
|
reply({:error, message}, state)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp idleSeconds(key) do
|
||||||
|
case Tracker.idleMs(Tracker, key, System.system_time(:millisecond)) do
|
||||||
|
nil -> 0
|
||||||
|
idleMs -> div(idleMs, 1000)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp withEntry(key, state, fun) do
|
||||||
|
case Store.encoding(key) do
|
||||||
|
nil -> reply({:error, "ERR no such key"}, state)
|
||||||
|
{:ok, entry} -> reply(fun.(entry), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodingName({:string, value, _ea}) do
|
||||||
|
cond do
|
||||||
|
integerString?(value) -> "int"
|
||||||
|
byte_size(value) <= 44 -> "embstr"
|
||||||
|
true -> "raw"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodingName({:list, value, _ea}) do
|
||||||
|
if compact?(value, length(value)), do: "listpack", else: "quicklist"
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodingName({:hash, value, _ea}) do
|
||||||
|
flat = Enum.flat_map(value, fn {field, item} -> [field, item] end)
|
||||||
|
if compact?(flat, map_size(value)), do: "listpack", else: "hashtable"
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodingName({:set, value, _ea}) do
|
||||||
|
members = MapSet.to_list(value)
|
||||||
|
size = MapSet.size(value)
|
||||||
|
|
||||||
|
cond do
|
||||||
|
size <= @intsetEntries and Enum.all?(members, &integerString?/1) -> "intset"
|
||||||
|
compact?(members, size) -> "listpack"
|
||||||
|
true -> "hashtable"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodingName({:zset, value, _ea}) do
|
||||||
|
members = value |> SkipList.to_list() |> Enum.map(fn {_score, member} -> member end)
|
||||||
|
if compact?(members, SkipList.size(value)), do: "listpack", else: "skiplist"
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodingName({:stream, _value, _ea}), do: "stream"
|
||||||
|
|
||||||
|
defp compact?(values, size) do
|
||||||
|
size <= @listpackEntries and Enum.all?(values, &(byte_size(&1) <= @listpackValue))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp refcount({:string, value, _ea}) do
|
||||||
|
if integerString?(value) and String.to_integer(value) in 0..9999 do
|
||||||
|
@sharedRefcount
|
||||||
|
else
|
||||||
|
1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp refcount(_entry), do: 1
|
||||||
|
|
||||||
|
defp integerString?(value) do
|
||||||
|
case Integer.parse(value) do
|
||||||
|
{number, ""} -> Integer.to_string(number) == value and byte_size(value) <= 20
|
||||||
|
_ -> false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseIntercard(numkeysRaw, rest) do
|
||||||
|
case Integer.parse(numkeysRaw) do
|
||||||
|
{numkeys, ""} when numkeys <= 0 ->
|
||||||
|
{:reply, {:error, "ERR numkeys should be greater than 0"}}
|
||||||
|
|
||||||
|
{numkeys, ""} when numkeys > length(rest) ->
|
||||||
|
{:reply, {:error, "ERR Number of keys can't be greater than number of args"}}
|
||||||
|
|
||||||
|
{numkeys, ""} ->
|
||||||
|
{keys, tail} = Enum.split(rest, numkeys)
|
||||||
|
|
||||||
|
case parseIntercardLimit(tail) do
|
||||||
|
{:ok, limit} -> {:ok, keys, limit}
|
||||||
|
{:reply, term} -> {:reply, term}
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseIntercardLimit([]), do: {:ok, 0}
|
||||||
|
|
||||||
|
defp parseIntercardLimit([token, raw]) do
|
||||||
|
if String.upcase(token) == "LIMIT" do
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{limit, ""} when limit >= 0 -> {:ok, limit}
|
||||||
|
{_negative, ""} -> {:reply, {:error, "ERR LIMIT can't be negative"}}
|
||||||
|
_ -> {:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
else
|
||||||
|
{:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseIntercardLimit(_tail), do: {:reply, Reply.syntax()}
|
||||||
|
|
||||||
|
defp parseCopyOptions([], replace), do: {:ok, replace}
|
||||||
|
|
||||||
|
defp parseCopyOptions([token | rest], replace) do
|
||||||
|
case {String.upcase(token), rest} do
|
||||||
|
{"REPLACE", tail} ->
|
||||||
|
parseCopyOptions(tail, true)
|
||||||
|
|
||||||
|
{"DB", [indexRaw | tail]} ->
|
||||||
|
case Integer.parse(indexRaw) do
|
||||||
|
{0, ""} -> parseCopyOptions(tail, replace)
|
||||||
|
{_other, ""} -> {:reply, {:error, "ERR DB index is out of range"}}
|
||||||
|
_ -> {:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseExpireFlags(tokens) do
|
||||||
|
tokens
|
||||||
|
|> Enum.reduce_while({:ok, %{nx: false, xx: false, gt: false, lt: false}}, fn token,
|
||||||
|
{:ok, flags} ->
|
||||||
|
case String.upcase(token) do
|
||||||
|
"NX" -> {:cont, {:ok, %{flags | nx: true}}}
|
||||||
|
"XX" -> {:cont, {:ok, %{flags | xx: true}}}
|
||||||
|
"GT" -> {:cont, {:ok, %{flags | gt: true}}}
|
||||||
|
"LT" -> {:cont, {:ok, %{flags | lt: true}}}
|
||||||
|
other -> {:halt, {:reply, {:error, "ERR Unsupported option #{other}"}}}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> validateExpireFlags()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp validateExpireFlags({:reply, term}), do: {:reply, term}
|
||||||
|
|
||||||
|
defp validateExpireFlags({:ok, flags}) do
|
||||||
|
incompatible = (flags.nx and (flags.xx or flags.gt or flags.lt)) or (flags.gt and flags.lt)
|
||||||
|
|
||||||
|
if incompatible do
|
||||||
|
{:reply, {:error, "ERR NX and XX, GT or LT options at the same time are not compatible"}}
|
||||||
|
else
|
||||||
|
{:ok, flags}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp expireAt(key, msAbs, flags) do
|
||||||
|
case Store.expiretime(key, :milliseconds) do
|
||||||
|
-2 -> 0
|
||||||
|
current -> if expireAllowed?(flags, current, msAbs), do: Store.expireat(key, msAbs), else: 0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp expireAllowed?(flags, current, msAbs) do
|
||||||
|
cond do
|
||||||
|
flags.nx -> current == -1
|
||||||
|
flags.xx and current == -1 -> false
|
||||||
|
flags.gt -> current != -1 and msAbs > current
|
||||||
|
flags.lt -> current == -1 or msAbs < current
|
||||||
|
true -> true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp zrevrange(key, startRaw, stopRaw, withscores, state) do
|
||||||
|
Reply.with_two_ints(startRaw, stopRaw, state, fn start, stop ->
|
||||||
|
Reply.zrange_reply(Store.zrevrange(key, start, stop), withscores)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp scoreEntry(nil), do: Reply.map_score(nil)
|
||||||
|
defp scoreEntry(score), do: Reply.map_score({:ok, score})
|
||||||
|
|
||||||
|
defp withType(key, expected, state, fun) do
|
||||||
|
case Store.type(key) do
|
||||||
|
"none" -> reply(fun.(), state)
|
||||||
|
^expected -> reply(fun.(), state)
|
||||||
|
_ -> reply(Reply.wrongtype(), state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp singleReply({:ok, [value | _rest]}), do: {:bulk, value}
|
||||||
|
defp singleReply({:ok, []}), do: {:bulk, nil}
|
||||||
|
defp singleReply({:error, reason}), do: errorTerm(reason)
|
||||||
|
|
||||||
|
defp arrayReply({:ok, values}), do: {:array, Reply.bulks(values)}
|
||||||
|
defp arrayReply({:error, reason}), do: errorTerm(reason)
|
||||||
|
|
||||||
|
defp integerReply({:ok, value}), do: {:integer, value}
|
||||||
|
defp integerReply({:error, reason}), do: errorTerm(reason)
|
||||||
|
|
||||||
|
defp bulkReply({:ok, value}), do: {:bulk, value}
|
||||||
|
defp bulkReply(nil), do: {:bulk, nil}
|
||||||
|
defp bulkReply({:error, reason}), do: errorTerm(reason)
|
||||||
|
|
||||||
|
defp rankReply({:ok, rank}), do: {:integer, rank}
|
||||||
|
defp rankReply(nil), do: {:bulk, nil}
|
||||||
|
defp rankReply({:error, reason}), do: errorTerm(reason)
|
||||||
|
|
||||||
|
defp statusReply({:ok, "OK"}), do: {:simple, "OK"}
|
||||||
|
defp statusReply({:error, reason}), do: errorTerm(reason)
|
||||||
|
|
||||||
|
defp errorTerm(:wrongtype), do: Reply.wrongtype()
|
||||||
|
defp errorTerm(:not_integer), do: Reply.not_integer()
|
||||||
|
defp errorTerm(:not_float), do: Reply.not_float()
|
||||||
|
defp errorTerm(:nokey), do: {:error, "ERR no such key"}
|
||||||
|
defp errorTerm(:range), do: {:error, "ERR index out of range"}
|
||||||
|
|
||||||
|
defp reply(term, state), do: {[term], state}
|
||||||
|
end
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
defmodule Excache.Command.Keyspace.Scan do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Command.Reply
|
||||||
|
alias Excache.Store
|
||||||
|
alias Excache.Store.Ops.Extra
|
||||||
|
|
||||||
|
@defaultCount 10
|
||||||
|
@buckets 64
|
||||||
|
|
||||||
|
def dispatch("SCAN", [cursorRaw | rest], state) do
|
||||||
|
with {:ok, cursor} <- parseCursor(cursorRaw),
|
||||||
|
{:ok, options} <- parseOptions(rest, [:match, :count, :type], defaults()) do
|
||||||
|
{next, keys} = walkShards(cursor, options)
|
||||||
|
reply(cursorReply(next, Reply.bulks(keys)), state)
|
||||||
|
else
|
||||||
|
{:reply, term} -> reply(term, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def dispatch("HSCAN", [key, cursorRaw | rest], state),
|
||||||
|
do: collection(:hash, key, cursorRaw, rest, state)
|
||||||
|
|
||||||
|
def dispatch("SSCAN", [key, cursorRaw | rest], state),
|
||||||
|
do: collection(:set, key, cursorRaw, rest, state)
|
||||||
|
|
||||||
|
def dispatch("ZSCAN", [key, cursorRaw | rest], state),
|
||||||
|
do: collection(:zset, key, cursorRaw, rest, state)
|
||||||
|
|
||||||
|
def dispatch(_name, _args, _state), do: :unhandled
|
||||||
|
|
||||||
|
defp collection(kind, key, cursorRaw, rest, state) do
|
||||||
|
with {:ok, cursor} <- parseCursor(cursorRaw),
|
||||||
|
{:ok, options} <- parseOptions(rest, allowedOptions(kind), defaults()),
|
||||||
|
{:ok, elements} <- fetchElements(kind, key) do
|
||||||
|
{next, selected} = walkBuckets(elements, cursor, options)
|
||||||
|
reply(cursorReply(next, renderElements(kind, selected, options)), state)
|
||||||
|
else
|
||||||
|
{:reply, term} -> reply(term, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp allowedOptions(:hash), do: [:match, :count, :novalues]
|
||||||
|
defp allowedOptions(_kind), do: [:match, :count]
|
||||||
|
|
||||||
|
defp fetchElements(:hash, key) do
|
||||||
|
case Store.hgetall(key) do
|
||||||
|
{:ok, flat} ->
|
||||||
|
{:ok, flat |> Enum.chunk_every(2) |> Enum.map(fn [field, value] -> {field, value} end)}
|
||||||
|
|
||||||
|
{:error, :wrongtype} ->
|
||||||
|
{:reply, Reply.wrongtype()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fetchElements(:set, key) do
|
||||||
|
case Store.smembers(key) do
|
||||||
|
{:ok, members} -> {:ok, Enum.map(members, &{&1, nil})}
|
||||||
|
{:error, :wrongtype} -> {:reply, Reply.wrongtype()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fetchElements(:zset, key) do
|
||||||
|
case Store.zrange(key, 0, -1) do
|
||||||
|
{:ok, entries} -> {:ok, Enum.map(entries, fn {score, member} -> {member, score} end)}
|
||||||
|
{:error, :wrongtype} -> {:reply, Reply.wrongtype()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp renderElements(:hash, elements, %{novalues: true}),
|
||||||
|
do: Enum.map(elements, fn {field, _value} -> {:bulk, field} end)
|
||||||
|
|
||||||
|
defp renderElements(:hash, elements, _options),
|
||||||
|
do: Enum.flat_map(elements, fn {field, value} -> [{:bulk, field}, {:bulk, value}] end)
|
||||||
|
|
||||||
|
defp renderElements(:set, elements, _options),
|
||||||
|
do: Enum.map(elements, fn {member, _value} -> {:bulk, member} end)
|
||||||
|
|
||||||
|
defp renderElements(:zset, elements, _options) do
|
||||||
|
Enum.flat_map(elements, fn {member, score} ->
|
||||||
|
[{:bulk, member}, Reply.map_score({:ok, score})]
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp walkShards(cursor, options) do
|
||||||
|
if cursor >= Store.shard_count() do
|
||||||
|
{0, []}
|
||||||
|
else
|
||||||
|
consumeShards(cursor, options, [])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp consumeShards(cursor, options, acc) do
|
||||||
|
{next, keys} = Store.scan(cursor, options.match, options.type)
|
||||||
|
gathered = acc ++ keys
|
||||||
|
|
||||||
|
cond do
|
||||||
|
next == 0 -> {0, gathered}
|
||||||
|
length(gathered) >= options.count -> {next, gathered}
|
||||||
|
true -> consumeShards(next, options, gathered)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp walkBuckets(_elements, cursor, _options) when cursor >= @buckets, do: {0, []}
|
||||||
|
|
||||||
|
defp walkBuckets(elements, cursor, options) do
|
||||||
|
grouped = Enum.group_by(elements, fn {member, _value} -> bucketOf(member) end)
|
||||||
|
consumeBuckets(grouped, cursor, options, [], 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp consumeBuckets(grouped, bucket, options, acc, examined) do
|
||||||
|
chunk = Map.get(grouped, bucket, [])
|
||||||
|
selected = acc ++ matching(chunk, options.match)
|
||||||
|
seen = examined + length(chunk)
|
||||||
|
next = bucket + 1
|
||||||
|
|
||||||
|
cond do
|
||||||
|
next >= @buckets -> {0, selected}
|
||||||
|
seen >= options.count -> {next, selected}
|
||||||
|
true -> consumeBuckets(grouped, next, options, selected, seen)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp bucketOf(member), do: rem(:erlang.phash2(member), @buckets)
|
||||||
|
|
||||||
|
defp matching(chunk, nil), do: chunk
|
||||||
|
|
||||||
|
defp matching(chunk, regex),
|
||||||
|
do: Enum.filter(chunk, fn {member, _value} -> Regex.match?(regex, member) end)
|
||||||
|
|
||||||
|
defp parseCursor(raw) do
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{cursor, ""} when cursor >= 0 -> {:ok, cursor}
|
||||||
|
_ -> {:reply, {:error, "ERR invalid cursor"}}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp defaults, do: %{match: nil, count: @defaultCount, type: nil, novalues: false}
|
||||||
|
|
||||||
|
defp parseOptions([], _allowed, options), do: {:ok, options}
|
||||||
|
|
||||||
|
defp parseOptions([token | rest], allowed, options) do
|
||||||
|
case {String.upcase(token), rest} do
|
||||||
|
{"MATCH", [pattern | tail]} ->
|
||||||
|
withOption(:match, allowed, fn ->
|
||||||
|
parseOptions(tail, allowed, %{options | match: Extra.compilePattern(pattern)})
|
||||||
|
end)
|
||||||
|
|
||||||
|
{"COUNT", [raw | tail]} ->
|
||||||
|
withOption(:count, allowed, fn -> parseCount(raw, tail, allowed, options) end)
|
||||||
|
|
||||||
|
{"TYPE", [raw | tail]} ->
|
||||||
|
withOption(:type, allowed, fn ->
|
||||||
|
parseOptions(tail, allowed, %{options | type: String.downcase(raw)})
|
||||||
|
end)
|
||||||
|
|
||||||
|
{"NOVALUES", tail} ->
|
||||||
|
withOption(:novalues, allowed, fn ->
|
||||||
|
parseOptions(tail, allowed, %{options | novalues: true})
|
||||||
|
end)
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp withOption(name, allowed, fun) do
|
||||||
|
if name in allowed, do: fun.(), else: {:reply, Reply.syntax()}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parseCount(raw, tail, allowed, options) do
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{count, ""} when count > 0 -> parseOptions(tail, allowed, %{options | count: count})
|
||||||
|
{_tooSmall, ""} -> {:reply, Reply.syntax()}
|
||||||
|
_ -> {:reply, Reply.not_integer()}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp cursorReply(cursor, items),
|
||||||
|
do: {:array, [{:bulk, Integer.to_string(cursor)}, {:array, items}]}
|
||||||
|
|
||||||
|
defp reply(term, state), do: {[term], state}
|
||||||
|
end
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
defmodule Excache.Command.Server do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Command.Reply
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Memory
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@doctorHeavyKey 25
|
||||||
|
@doctorNearLimit 80
|
||||||
|
|
||||||
|
def dispatch("MEMORY", [subcommand | rest], state),
|
||||||
|
do: memory(String.upcase(subcommand), rest, state)
|
||||||
|
|
||||||
|
def dispatch("MEMORY", [], state), do: {[wrongArity("memory")], state}
|
||||||
|
|
||||||
|
def dispatch("CONFIG", [subcommand | rest], state),
|
||||||
|
do: config(String.upcase(subcommand), rest, state)
|
||||||
|
|
||||||
|
def dispatch("CONFIG", [], state), do: {[wrongArity("config")], state}
|
||||||
|
|
||||||
|
defp memory("USAGE", [key], state), do: {[usage(key)], state}
|
||||||
|
|
||||||
|
defp memory("USAGE", [key, option, count], state) do
|
||||||
|
cond do
|
||||||
|
String.upcase(option) != "SAMPLES" -> {[Reply.syntax()], state}
|
||||||
|
not nonNegativeInteger?(count) -> {[Reply.not_integer()], state}
|
||||||
|
true -> {[usage(key)], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp memory("STATS", [], state), do: {[{:array, statsArray()}], state}
|
||||||
|
|
||||||
|
defp memory("DOCTOR", [], state), do: {[{:bulk, doctor()}], state}
|
||||||
|
|
||||||
|
defp memory(subcommand, _args, state), do: {[unknownSubcommand("MEMORY", subcommand)], state}
|
||||||
|
|
||||||
|
defp config("GET", [_ | _] = patterns, state) do
|
||||||
|
values =
|
||||||
|
patterns
|
||||||
|
|> Enum.flat_map(&Config.getGlob/1)
|
||||||
|
|> Enum.uniq_by(fn {name, _value} -> name end)
|
||||||
|
|> Enum.sort()
|
||||||
|
|> Enum.flat_map(fn {name, value} -> [{:bulk, name}, {:bulk, value}] end)
|
||||||
|
|
||||||
|
{[{:array, values}], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp config("GET", [], state), do: {[wrongArity("config|get")], state}
|
||||||
|
|
||||||
|
defp config("SET", [_, _ | _] = args, state) do
|
||||||
|
if rem(length(args), 2) == 0 do
|
||||||
|
{[applySettings(Reply.pairs(args))], state}
|
||||||
|
else
|
||||||
|
{[wrongArity("config|set")], state}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp config("SET", _args, state), do: {[wrongArity("config|set")], state}
|
||||||
|
|
||||||
|
defp config("RESETSTAT", [], state) do
|
||||||
|
Runner.resetStats()
|
||||||
|
{[{:simple, "OK"}], state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp config(subcommand, _args, state), do: {[unknownSubcommand("CONFIG", subcommand)], state}
|
||||||
|
|
||||||
|
defp applySettings(pairs) do
|
||||||
|
case Config.setMany(pairs) do
|
||||||
|
:ok -> {:simple, "OK"}
|
||||||
|
{:error, name, reason} -> settingError(name, reason)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp settingError(name, :unknown),
|
||||||
|
do: {:error, "ERR Unknown option or number of arguments for CONFIG SET - '#{name}'"}
|
||||||
|
|
||||||
|
defp settingError(name, :immutable),
|
||||||
|
do: {:error, "ERR CONFIG SET failed - can't set immutable config '#{name}'"}
|
||||||
|
|
||||||
|
defp settingError(_name, {:invalid, hint}), do: {:error, "ERR CONFIG SET failed - #{hint}"}
|
||||||
|
|
||||||
|
defp usage(key) do
|
||||||
|
case Memory.usage(key) do
|
||||||
|
nil -> {:bulk, nil}
|
||||||
|
bytes -> {:integer, bytes}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp statsArray do
|
||||||
|
stats = Memory.stats(Store.dump())
|
||||||
|
runner = Runner.stats()
|
||||||
|
|
||||||
|
stats
|
||||||
|
|> statsPairs(runner)
|
||||||
|
|> Enum.flat_map(fn {name, term} -> [{:bulk, name}, term] end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp statsPairs(stats, runner) do
|
||||||
|
[
|
||||||
|
{"keys.count", {:integer, stats.keys}},
|
||||||
|
{"keys.volatile", {:integer, stats.volatileKeys}},
|
||||||
|
{"keys.bytes-per-key", {:integer, stats.bytesPerKey}},
|
||||||
|
{"total.allocated", {:integer, stats.usedMemory}},
|
||||||
|
{"dataset.bytes", {:integer, stats.datasetBytes}},
|
||||||
|
{"overhead.total", {:integer, stats.overheadBytes}},
|
||||||
|
{"largest.key", {:bulk, largestName(stats.largestKey)}},
|
||||||
|
{"largest.key.bytes", {:integer, largestBytes(stats.largestKey)}},
|
||||||
|
{"maxmemory", {:integer, Config.maxmemory()}},
|
||||||
|
{"maxmemory.policy", {:bulk, Eviction.policyName(Config.policy())}},
|
||||||
|
{"maxmemory.samples", {:integer, Config.maxmemorySamples()}},
|
||||||
|
{"evicted.keys", {:integer, runner.evictedKeys}},
|
||||||
|
{"oom.rejections", {:integer, runner.oomRejections}},
|
||||||
|
{"tracker.keys", {:integer, Tracker.size(Tracker)}}
|
||||||
|
] ++ typePairs(stats.byType)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp typePairs(byType) do
|
||||||
|
Enum.flat_map(Memory.kinds(), fn kind ->
|
||||||
|
usage = Map.fetch!(byType, kind)
|
||||||
|
|
||||||
|
[
|
||||||
|
{"type.#{kind}.keys", {:integer, usage.keys}},
|
||||||
|
{"type.#{kind}.bytes", {:integer, usage.bytes}}
|
||||||
|
]
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp largestName(nil), do: nil
|
||||||
|
defp largestName({key, _bytes}), do: key
|
||||||
|
|
||||||
|
defp largestBytes(nil), do: 0
|
||||||
|
defp largestBytes({_key, bytes}), do: bytes
|
||||||
|
|
||||||
|
defp doctor do
|
||||||
|
entries = Store.dump()
|
||||||
|
stats = Memory.stats(entries)
|
||||||
|
|
||||||
|
if stats.keys == 0 do
|
||||||
|
"Excache holds no keys, there is nothing to report."
|
||||||
|
else
|
||||||
|
report = %{
|
||||||
|
stats: stats,
|
||||||
|
limit: Config.maxmemory(),
|
||||||
|
policy: Config.policy(),
|
||||||
|
runner: Runner.stats(),
|
||||||
|
tracked: Tracker.size(Tracker)
|
||||||
|
}
|
||||||
|
|
||||||
|
case findings(report) do
|
||||||
|
[] -> headline(report) <> " Nothing looks wrong."
|
||||||
|
found -> Enum.join([headline(report) | found], "\n")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp headline(%{stats: stats, policy: policy, limit: limit}) do
|
||||||
|
limitText = if limit == 0, do: "no limit", else: "#{limit} byte limit"
|
||||||
|
|
||||||
|
"#{stats.keys} keys, #{stats.usedMemory} bytes used, #{limitText}, " <>
|
||||||
|
"maxmemory-policy #{Eviction.policyName(policy)}."
|
||||||
|
end
|
||||||
|
|
||||||
|
defp findings(report) do
|
||||||
|
[
|
||||||
|
&noLimit/1,
|
||||||
|
&overLimit/1,
|
||||||
|
&nearLimit/1,
|
||||||
|
&noVolatileKeys/1,
|
||||||
|
&dominantKey/1,
|
||||||
|
&overheadHeavy/1,
|
||||||
|
&untrackedKeys/1,
|
||||||
|
&evictionActivity/1,
|
||||||
|
&refusedWrites/1
|
||||||
|
]
|
||||||
|
|> Enum.map(& &1.(report))
|
||||||
|
|> Enum.reject(&is_nil/1)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp noLimit(%{limit: 0, stats: stats}) do
|
||||||
|
"maxmemory is not set, so nothing bounds the keyspace; it already holds " <>
|
||||||
|
"#{stats.usedMemory} bytes and will keep growing until the host runs out."
|
||||||
|
end
|
||||||
|
|
||||||
|
defp noLimit(_report), do: nil
|
||||||
|
|
||||||
|
defp overLimit(%{limit: limit, stats: stats, policy: :noeviction}) when limit > 0 do
|
||||||
|
if stats.usedMemory > limit do
|
||||||
|
"used memory #{stats.usedMemory} is over the #{limit} byte limit and " <>
|
||||||
|
"maxmemory-policy is noeviction, so every command that allocates is being refused."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp overLimit(%{limit: limit, stats: stats}) when limit > 0 do
|
||||||
|
if stats.usedMemory > limit do
|
||||||
|
"used memory #{stats.usedMemory} is over the #{limit} byte limit, " <>
|
||||||
|
"the next write will free keys."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp overLimit(_report), do: nil
|
||||||
|
|
||||||
|
defp nearLimit(%{limit: limit, stats: stats}) when limit > 0 do
|
||||||
|
share = percent(stats.usedMemory, limit)
|
||||||
|
|
||||||
|
if share >= @doctorNearLimit and stats.usedMemory <= limit do
|
||||||
|
"used memory is #{share} percent of the #{limit} byte limit, eviction is close."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp nearLimit(_report), do: nil
|
||||||
|
|
||||||
|
defp noVolatileKeys(%{policy: policy, stats: stats, limit: limit}) when limit > 0 do
|
||||||
|
if Eviction.volatile?(policy) and stats.volatileKeys == 0 do
|
||||||
|
"maxmemory-policy #{Eviction.policyName(policy)} only evicts keys with a TTL and " <>
|
||||||
|
"none of the #{stats.keys} keys has one, so eviction cannot free anything."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp noVolatileKeys(_report), do: nil
|
||||||
|
|
||||||
|
defp dominantKey(%{stats: %{largestKey: {key, bytes}} = stats}) do
|
||||||
|
share = percent(bytes, stats.usedMemory)
|
||||||
|
|
||||||
|
if share >= @doctorHeavyKey and stats.keys > 1 do
|
||||||
|
"key '#{key}' holds #{bytes} bytes, #{share} percent of everything stored."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp dominantKey(_report), do: nil
|
||||||
|
|
||||||
|
defp overheadHeavy(%{stats: stats}) do
|
||||||
|
share = percent(stats.overheadBytes, stats.usedMemory)
|
||||||
|
|
||||||
|
if share > 50 do
|
||||||
|
"per key overhead is #{share} percent of the total at #{stats.bytesPerKey} bytes per key, " <>
|
||||||
|
"the keyspace is dominated by very small values."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp overheadHeavy(_report), do: nil
|
||||||
|
|
||||||
|
defp untrackedKeys(%{policy: policy, stats: stats, tracked: tracked}) do
|
||||||
|
if policy in [:allkeysLru, :allkeysLfu, :volatileLru, :volatileLfu] and tracked < stats.keys do
|
||||||
|
"access metadata covers #{tracked} of #{stats.keys} keys, " <>
|
||||||
|
"the keys nobody has read yet are the first to go."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp untrackedKeys(_report), do: nil
|
||||||
|
|
||||||
|
defp evictionActivity(%{runner: %{evictedKeys: evicted}}) when evicted > 0,
|
||||||
|
do: "#{evicted} keys have been evicted since the last CONFIG RESETSTAT."
|
||||||
|
|
||||||
|
defp evictionActivity(_report), do: nil
|
||||||
|
|
||||||
|
defp refusedWrites(%{runner: %{oomRejections: rejected}}) when rejected > 0,
|
||||||
|
do: "#{rejected} commands hit the memory limit since the last CONFIG RESETSTAT."
|
||||||
|
|
||||||
|
defp refusedWrites(_report), do: nil
|
||||||
|
|
||||||
|
defp percent(_part, 0), do: 0
|
||||||
|
defp percent(part, whole), do: round(part * 100 / whole)
|
||||||
|
|
||||||
|
defp nonNegativeInteger?(raw) do
|
||||||
|
match?({value, ""} when value >= 0, Integer.parse(raw))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp wrongArity(name),
|
||||||
|
do: {:error, "ERR wrong number of arguments for '#{name}' command"}
|
||||||
|
|
||||||
|
defp unknownSubcommand(command, subcommand) do
|
||||||
|
{:error,
|
||||||
|
"ERR Unknown subcommand or wrong number of arguments for '#{subcommand}'. " <>
|
||||||
|
"Try #{command} HELP."}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
defmodule Excache.Config do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Eviction
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
alias Excache.Persistence
|
||||||
|
|
||||||
|
@defaults %{
|
||||||
|
maxmemory: 0,
|
||||||
|
maxmemoryPolicy: :noeviction,
|
||||||
|
maxmemorySamples: 5,
|
||||||
|
lfuLogFactor: 10,
|
||||||
|
lfuDecayTime: 1,
|
||||||
|
port: 6379,
|
||||||
|
snapshot_path: nil,
|
||||||
|
aof_path: nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@parameters %{
|
||||||
|
"maxmemory" => %{env: :maxmemory, type: :bytes, mutable: true, runtime: true},
|
||||||
|
"maxmemory-policy" => %{env: :maxmemoryPolicy, type: :policy, mutable: true, runtime: true},
|
||||||
|
"maxmemory-samples" => %{
|
||||||
|
env: :maxmemorySamples,
|
||||||
|
type: {:bounded, 1, 64},
|
||||||
|
mutable: true,
|
||||||
|
runtime: true
|
||||||
|
},
|
||||||
|
"lfu-log-factor" => %{
|
||||||
|
env: :lfuLogFactor,
|
||||||
|
type: {:bounded, 0, 255},
|
||||||
|
mutable: true,
|
||||||
|
runtime: true
|
||||||
|
},
|
||||||
|
"lfu-decay-time" => %{
|
||||||
|
env: :lfuDecayTime,
|
||||||
|
type: {:bounded, 0, 1440},
|
||||||
|
mutable: true,
|
||||||
|
runtime: true
|
||||||
|
},
|
||||||
|
"port" => %{env: :port, type: {:bounded, 0, 65_535}, mutable: false, runtime: false},
|
||||||
|
"snapshot_path" => %{env: :snapshot_path, type: :path, mutable: true, runtime: false},
|
||||||
|
"aof_path" => %{env: :aof_path, type: :path, mutable: false, runtime: false}
|
||||||
|
}
|
||||||
|
|
||||||
|
@trackedPolicies [:allkeysLru, :allkeysLfu, :volatileLru, :volatileLfu]
|
||||||
|
|
||||||
|
@units %{
|
||||||
|
"" => 1,
|
||||||
|
"b" => 1,
|
||||||
|
"k" => 1000,
|
||||||
|
"kb" => 1024,
|
||||||
|
"m" => 1_000_000,
|
||||||
|
"mb" => 1_048_576,
|
||||||
|
"g" => 1_000_000_000,
|
||||||
|
"gb" => 1_073_741_824
|
||||||
|
}
|
||||||
|
|
||||||
|
def names, do: @parameters |> Map.keys() |> Enum.sort()
|
||||||
|
|
||||||
|
def get(name) do
|
||||||
|
case Map.fetch(@parameters, String.downcase(name)) do
|
||||||
|
{:ok, spec} -> {:ok, readSpec(spec)}
|
||||||
|
:error -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def set(name, raw) do
|
||||||
|
case prepare(name, raw) do
|
||||||
|
{:ok, change} -> commit(change)
|
||||||
|
{:error, reason} -> {:error, reason}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def setMany(pairs) do
|
||||||
|
pairs
|
||||||
|
|> Enum.reduce_while({:ok, []}, fn {name, raw}, {:ok, acc} ->
|
||||||
|
case prepare(name, raw) do
|
||||||
|
{:ok, change} -> {:cont, {:ok, [change | acc]}}
|
||||||
|
{:error, reason} -> {:halt, {:error, name, reason}}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> case do
|
||||||
|
{:ok, changes} ->
|
||||||
|
changes |> Enum.reverse() |> Enum.each(&commit/1)
|
||||||
|
:ok
|
||||||
|
|
||||||
|
failure ->
|
||||||
|
failure
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def getGlob(pattern) do
|
||||||
|
chars = pattern |> String.downcase() |> String.to_charlist()
|
||||||
|
|
||||||
|
for name <- names(), matches?(chars, String.to_charlist(name)) do
|
||||||
|
{:ok, value} = get(name)
|
||||||
|
{name, format(value)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def reset do
|
||||||
|
for {_name, spec} <- @parameters, spec.runtime do
|
||||||
|
Application.delete_env(:excache, spec.env)
|
||||||
|
end
|
||||||
|
|
||||||
|
Runner.invalidate()
|
||||||
|
Runner.reloadTracker()
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
defp format(nil), do: ""
|
||||||
|
defp format(value) when is_integer(value), do: Integer.to_string(value)
|
||||||
|
defp format(value) when is_binary(value), do: value
|
||||||
|
defp format(policy) when is_atom(policy), do: Eviction.policyName(policy)
|
||||||
|
|
||||||
|
def maxmemory, do: readEnv(:maxmemory)
|
||||||
|
def policy, do: readEnv(:maxmemoryPolicy)
|
||||||
|
def maxmemorySamples, do: readEnv(:maxmemorySamples)
|
||||||
|
def lfuLogFactor, do: readEnv(:lfuLogFactor)
|
||||||
|
def lfuDecayTime, do: readEnv(:lfuDecayTime)
|
||||||
|
|
||||||
|
def tracksAccess?, do: policy() in @trackedPolicies
|
||||||
|
|
||||||
|
defp hint(:bytes), do: "argument must be a memory value like 512, 64kb, 100mb or 1gb"
|
||||||
|
|
||||||
|
defp hint(:policy),
|
||||||
|
do:
|
||||||
|
"argument must be one of the following: " <>
|
||||||
|
Enum.map_join(Eviction.policies(), ", ", &Eviction.policyName/1)
|
||||||
|
|
||||||
|
defp hint({:bounded, low, high}), do: "argument must be an integer between #{low} and #{high}"
|
||||||
|
defp hint(:path), do: "argument must be a non empty path"
|
||||||
|
|
||||||
|
defp prepare(name, raw) do
|
||||||
|
case Map.fetch(@parameters, String.downcase(name)) do
|
||||||
|
:error ->
|
||||||
|
{:error, :unknown}
|
||||||
|
|
||||||
|
{:ok, %{mutable: false}} ->
|
||||||
|
{:error, :immutable}
|
||||||
|
|
||||||
|
{:ok, spec} ->
|
||||||
|
case parse(spec.type, raw) do
|
||||||
|
{:ok, value} -> {:ok, {spec, value}}
|
||||||
|
:error -> {:error, {:invalid, hint(spec.type)}}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp commit({spec, value}) do
|
||||||
|
previous = readSpec(spec)
|
||||||
|
Application.put_env(:excache, spec.env, value)
|
||||||
|
if value != previous, do: applyChange(spec)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
defp applyChange(%{env: env}) when env in [:lfuLogFactor, :lfuDecayTime],
|
||||||
|
do: Runner.reloadTracker()
|
||||||
|
|
||||||
|
defp applyChange(%{env: env}) when env in [:maxmemory, :maxmemoryPolicy, :maxmemorySamples],
|
||||||
|
do: Runner.invalidate()
|
||||||
|
|
||||||
|
defp applyChange(_spec), do: :ok
|
||||||
|
|
||||||
|
defp parse(:bytes, raw), do: parseBytes(raw)
|
||||||
|
defp parse(:policy, raw), do: Eviction.parsePolicy(String.downcase(raw))
|
||||||
|
|
||||||
|
defp parse({:bounded, low, high}, raw) do
|
||||||
|
case Integer.parse(raw) do
|
||||||
|
{value, ""} when value >= low and value <= high -> {:ok, value}
|
||||||
|
_ -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parse(:path, ""), do: :error
|
||||||
|
defp parse(:path, raw), do: {:ok, raw}
|
||||||
|
|
||||||
|
defp parseBytes(raw) do
|
||||||
|
{digits, unit} =
|
||||||
|
raw
|
||||||
|
|> String.downcase()
|
||||||
|
|> String.to_charlist()
|
||||||
|
|> Enum.split_while(&(&1 in ?0..?9))
|
||||||
|
|
||||||
|
with {:ok, factor} <- Map.fetch(@units, List.to_string(unit)),
|
||||||
|
{amount, ""} <- Integer.parse(List.to_string(digits)) do
|
||||||
|
{:ok, amount * factor}
|
||||||
|
else
|
||||||
|
_ -> :error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp readSpec(%{env: :snapshot_path}), do: Persistence.snapshot_path()
|
||||||
|
defp readSpec(%{env: :aof_path}), do: Aof.path()
|
||||||
|
defp readSpec(%{env: env}), do: readEnv(env)
|
||||||
|
|
||||||
|
defp readEnv(env), do: Application.get_env(:excache, env, @defaults[env])
|
||||||
|
|
||||||
|
defp matches?([], name), do: name == []
|
||||||
|
|
||||||
|
defp matches?([?* | pattern], name),
|
||||||
|
do: matches?(pattern, name) or (name != [] and matches?([?* | pattern], tl(name)))
|
||||||
|
|
||||||
|
defp matches?([?? | pattern], [_char | name]), do: matches?(pattern, name)
|
||||||
|
defp matches?([char | pattern], [char | name]), do: matches?(pattern, name)
|
||||||
|
defp matches?(_pattern, _name), do: false
|
||||||
|
end
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
defmodule Excache.Eviction do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
# Redis style maxmemory policies over the Excache keyspace.
|
||||||
|
#
|
||||||
|
# Nothing here sorts the whole keyspace. Every pass draws maxmemory-samples
|
||||||
|
# random candidates, scores them under the active policy and merges them into
|
||||||
|
# a bounded pool that survives between passes, so a strong candidate seen in
|
||||||
|
# an earlier pass still wins against a weak fresh sample. The victim is the
|
||||||
|
# pool head; the loop repeats until the estimated usage drops under the limit
|
||||||
|
# or the candidate set runs dry.
|
||||||
|
#
|
||||||
|
# Both the clock and the sampling function are injectable, so a run is fully
|
||||||
|
# reproducible. The default keyspace adapter reads the whole store once per
|
||||||
|
# run and then accounts freed bytes incrementally, the same estimate driven
|
||||||
|
# approach Redis uses inside performEvictions.
|
||||||
|
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Memory
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@policies [
|
||||||
|
:noeviction,
|
||||||
|
:allkeysLru,
|
||||||
|
:allkeysLfu,
|
||||||
|
:allkeysRandom,
|
||||||
|
:volatileLru,
|
||||||
|
:volatileLfu,
|
||||||
|
:volatileTtl,
|
||||||
|
:volatileRandom
|
||||||
|
]
|
||||||
|
|
||||||
|
@policyNames %{
|
||||||
|
noeviction: "noeviction",
|
||||||
|
allkeysLru: "allkeys-lru",
|
||||||
|
allkeysLfu: "allkeys-lfu",
|
||||||
|
allkeysRandom: "allkeys-random",
|
||||||
|
volatileLru: "volatile-lru",
|
||||||
|
volatileLfu: "volatile-lfu",
|
||||||
|
volatileTtl: "volatile-ttl",
|
||||||
|
volatileRandom: "volatile-random"
|
||||||
|
}
|
||||||
|
|
||||||
|
@volatilePolicies [:volatileLru, :volatileLfu, :volatileTtl, :volatileRandom]
|
||||||
|
@allkeysPolicies [:allkeysLru, :allkeysLfu, :allkeysRandom]
|
||||||
|
@randomPolicies [:allkeysRandom, :volatileRandom]
|
||||||
|
@lruPolicies [:allkeysLru, :volatileLru]
|
||||||
|
@lfuPolicies [:allkeysLfu, :volatileLfu]
|
||||||
|
|
||||||
|
@defaultSampleSize 5
|
||||||
|
@defaultPoolSize 16
|
||||||
|
@defaultSeed {5, 9, 27}
|
||||||
|
@lfuMaxCounter 255
|
||||||
|
|
||||||
|
def policies, do: @policies
|
||||||
|
|
||||||
|
def policyName(policy), do: Map.fetch!(@policyNames, policy)
|
||||||
|
|
||||||
|
def parsePolicy(name) when is_binary(name) do
|
||||||
|
case Enum.find(@policies, fn policy -> @policyNames[policy] == name end) do
|
||||||
|
nil -> :error
|
||||||
|
policy -> {:ok, policy}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def volatile?(policy), do: policy in @volatilePolicies
|
||||||
|
def allkeys?(policy), do: policy in @allkeysPolicies
|
||||||
|
def random?(policy), do: policy in @randomPolicies
|
||||||
|
|
||||||
|
def defaultSampleSize, do: @defaultSampleSize
|
||||||
|
def defaultPoolSize, do: @defaultPoolSize
|
||||||
|
|
||||||
|
def storeKeyspace do
|
||||||
|
%{entries: &Store.dump/0, delete: fn key -> Store.del([key]) end}
|
||||||
|
end
|
||||||
|
|
||||||
|
def candidates(entries, policy) do
|
||||||
|
entries
|
||||||
|
|> Enum.map(fn {key, kind, value, expiresAt} ->
|
||||||
|
%{
|
||||||
|
key: key,
|
||||||
|
kind: kind,
|
||||||
|
expiresAt: expiresAt,
|
||||||
|
bytes: Memory.estimateEntry(key, {kind, value, expiresAt})
|
||||||
|
}
|
||||||
|
end)
|
||||||
|
|> Enum.filter(&eligible?(&1, policy))
|
||||||
|
end
|
||||||
|
|
||||||
|
def selectVictim(entries, opts) do
|
||||||
|
config = configure(opts)
|
||||||
|
pool = candidates(entries, config.policy)
|
||||||
|
|
||||||
|
case pool do
|
||||||
|
[] ->
|
||||||
|
{:error, emptyReason(config.policy)}
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{victim, _pool, _config} = pickVictim(pool, [], config)
|
||||||
|
{:ok, victim.key}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def run(opts) do
|
||||||
|
config = configure(opts)
|
||||||
|
keyspace = Keyword.get_lazy(opts, :keyspace, &storeKeyspace/0)
|
||||||
|
entries = keyspace.entries.()
|
||||||
|
pool = candidates(entries, config.policy)
|
||||||
|
|
||||||
|
acc = %{used: Memory.usedMemory(entries), evicted: [], freed: 0, passes: 0}
|
||||||
|
|
||||||
|
cond do
|
||||||
|
config.maxmemory <= 0 or acc.used <= config.maxmemory ->
|
||||||
|
{:ok, report(config, acc, :withinLimit)}
|
||||||
|
|
||||||
|
config.policy == :noeviction ->
|
||||||
|
{:error, report(config, acc, :noeviction)}
|
||||||
|
|
||||||
|
pool == [] ->
|
||||||
|
{:error, report(config, acc, emptyReason(config.policy))}
|
||||||
|
|
||||||
|
true ->
|
||||||
|
evictLoop(pool, [], config, acc, keyspace)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp evictLoop(remaining, pool, config, acc, keyspace) do
|
||||||
|
cond do
|
||||||
|
acc.used <= config.maxmemory ->
|
||||||
|
{:ok, report(config, acc, :evicted)}
|
||||||
|
|
||||||
|
remaining == [] ->
|
||||||
|
{:error, report(config, acc, :exhausted)}
|
||||||
|
|
||||||
|
true ->
|
||||||
|
{victim, carried, config} = pickVictim(remaining, pool, config)
|
||||||
|
keyspace.delete.(victim.key)
|
||||||
|
if config.tracker, do: Tracker.forget(config.tracker, victim.key)
|
||||||
|
|
||||||
|
acc = %{
|
||||||
|
acc
|
||||||
|
| used: acc.used - victim.bytes,
|
||||||
|
freed: acc.freed + victim.bytes,
|
||||||
|
evicted: [victim.key | acc.evicted],
|
||||||
|
passes: acc.passes + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
nextRemaining = Enum.reject(remaining, &(&1.key == victim.key))
|
||||||
|
nextPool = Enum.reject(carried, fn {_score, held} -> held.key == victim.key end)
|
||||||
|
evictLoop(nextRemaining, nextPool, config, acc, keyspace)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp pickVictim(remaining, pool, config) do
|
||||||
|
{sampled, randState} = config.sampler.(remaining, config.sampleSize, config.randState)
|
||||||
|
config = %{config | randState: randState}
|
||||||
|
|
||||||
|
if random?(config.policy) do
|
||||||
|
{hd(sampled), pool, config}
|
||||||
|
else
|
||||||
|
metadata = config.metaFun.(Enum.map(sampled, & &1.key), config.now)
|
||||||
|
|
||||||
|
scored =
|
||||||
|
Enum.map(sampled, fn candidate ->
|
||||||
|
{score(config.policy, candidate, Map.get(metadata, candidate.key), config.now),
|
||||||
|
candidate}
|
||||||
|
end)
|
||||||
|
|
||||||
|
[{_score, victim} | rest] = mergePool(pool, scored, config.poolSize)
|
||||||
|
{victim, rest, config}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp mergePool(pool, scored, poolSize) do
|
||||||
|
(scored ++ pool)
|
||||||
|
|> Enum.uniq_by(fn {_score, candidate} -> candidate.key end)
|
||||||
|
|> Enum.sort_by(fn {candidateScore, _candidate} -> candidateScore end, :desc)
|
||||||
|
|> Enum.take(poolSize)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp score(policy, _candidate, meta, now) when policy in @lruPolicies, do: lruScore(meta, now)
|
||||||
|
|
||||||
|
defp score(policy, _candidate, meta, _now) when policy in @lfuPolicies,
|
||||||
|
do: @lfuMaxCounter - counterOf(meta)
|
||||||
|
|
||||||
|
defp score(:volatileTtl, candidate, _meta, _now), do: -candidate.expiresAt
|
||||||
|
|
||||||
|
defp lruScore(nil, _now), do: {1, 0}
|
||||||
|
defp lruScore(meta, now), do: {0, max(now - meta.lastAccess, 0)}
|
||||||
|
|
||||||
|
defp counterOf(nil), do: 0
|
||||||
|
defp counterOf(meta), do: meta.counter
|
||||||
|
|
||||||
|
defp eligible?(_candidate, :noeviction), do: false
|
||||||
|
|
||||||
|
defp eligible?(candidate, policy) do
|
||||||
|
if volatile?(policy), do: candidate.expiresAt != nil, else: true
|
||||||
|
end
|
||||||
|
|
||||||
|
defp emptyReason(policy) do
|
||||||
|
if volatile?(policy), do: :noVolatileKeys, else: :noCandidates
|
||||||
|
end
|
||||||
|
|
||||||
|
defp configure(opts) do
|
||||||
|
%{
|
||||||
|
policy: Keyword.fetch!(opts, :policy),
|
||||||
|
maxmemory: Keyword.get(opts, :maxmemory, 0),
|
||||||
|
now: Keyword.get_lazy(opts, :now, fn -> System.system_time(:millisecond) end),
|
||||||
|
sampleSize: Keyword.get(opts, :sampleSize, @defaultSampleSize),
|
||||||
|
poolSize: Keyword.get(opts, :poolSize, @defaultPoolSize),
|
||||||
|
sampler: Keyword.get(opts, :sampler, &sampleRandom/3),
|
||||||
|
randState: :rand.seed_s(:exsss, Keyword.get(opts, :seed, @defaultSeed)),
|
||||||
|
tracker: Keyword.get(opts, :tracker),
|
||||||
|
metaFun: metaFun(opts)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp metaFun(opts) do
|
||||||
|
cond do
|
||||||
|
Keyword.has_key?(opts, :metadata) ->
|
||||||
|
metadata = Keyword.fetch!(opts, :metadata)
|
||||||
|
fn keys, _now -> Map.take(metadata, keys) end
|
||||||
|
|
||||||
|
Keyword.has_key?(opts, :tracker) ->
|
||||||
|
tracker = Keyword.fetch!(opts, :tracker)
|
||||||
|
fn keys, now -> Tracker.metadata(tracker, keys, now) end
|
||||||
|
|
||||||
|
true ->
|
||||||
|
fn _keys, _now -> %{} end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp sampleRandom(pool, count, randState) when length(pool) <= count, do: {pool, randState}
|
||||||
|
|
||||||
|
defp sampleRandom(pool, count, randState) do
|
||||||
|
{taken, _rest, _size, nextState} =
|
||||||
|
Enum.reduce(1..count, {[], pool, length(pool), randState}, fn _index,
|
||||||
|
{acc, left, size, state} ->
|
||||||
|
{index, drawn} = :rand.uniform_s(size, state)
|
||||||
|
{picked, rest} = List.pop_at(left, index - 1)
|
||||||
|
{[picked | acc], rest, size - 1, drawn}
|
||||||
|
end)
|
||||||
|
|
||||||
|
{Enum.reverse(taken), nextState}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp report(config, acc, reason) do
|
||||||
|
%{
|
||||||
|
reason: reason,
|
||||||
|
policy: config.policy,
|
||||||
|
maxmemory: config.maxmemory,
|
||||||
|
usedMemory: acc.used,
|
||||||
|
freedBytes: acc.freed,
|
||||||
|
evicted: Enum.reverse(acc.evicted),
|
||||||
|
evictedCount: length(acc.evicted),
|
||||||
|
passes: acc.passes
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
defmodule Excache.Eviction.Runner do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
use GenServer
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Memory
|
||||||
|
alias Excache.Notify
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@meter {__MODULE__, :meter}
|
||||||
|
@used 1
|
||||||
|
@dirty 2
|
||||||
|
@checkedAt 3
|
||||||
|
@verdict 4
|
||||||
|
@slots 4
|
||||||
|
|
||||||
|
@defaultInterval 200
|
||||||
|
@defaultRecheck 20
|
||||||
|
|
||||||
|
def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
||||||
|
|
||||||
|
def admit do
|
||||||
|
limit = Config.maxmemory()
|
||||||
|
if limit <= 0, do: :ok, else: decide(limit)
|
||||||
|
end
|
||||||
|
|
||||||
|
def account(args) do
|
||||||
|
if Config.maxmemory() > 0, do: :counters.add(meter(), @dirty, argBytes(args))
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
def enforce, do: GenServer.call(__MODULE__, :enforce, 30_000)
|
||||||
|
|
||||||
|
def invalidate do
|
||||||
|
meter = meter()
|
||||||
|
:counters.put(meter, @dirty, 0)
|
||||||
|
:counters.put(meter, @checkedAt, 0)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
def reloadTracker, do: GenServer.call(__MODULE__, :reloadTracker)
|
||||||
|
|
||||||
|
def stats, do: GenServer.call(__MODULE__, :stats)
|
||||||
|
|
||||||
|
def resetStats, do: GenServer.call(__MODULE__, :resetStats)
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def init(_opts) do
|
||||||
|
:persistent_term.put(@meter, :counters.new(@slots, [:write_concurrency]))
|
||||||
|
schedule()
|
||||||
|
{:ok, %{evictedKeys: 0, oomRejections: 0}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:enforce, _from, state) do
|
||||||
|
{verdict, state} = sweep(state)
|
||||||
|
{:reply, verdict, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:reloadTracker, _from, state) do
|
||||||
|
wanted = %{logFactor: Config.lfuLogFactor(), decayTime: Config.lfuDecayTime()}
|
||||||
|
if Tracker.settings(Tracker) != wanted, do: restartTracker()
|
||||||
|
{:reply, :ok, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:stats, _from, state) do
|
||||||
|
{:reply, %{evictedKeys: state.evictedKeys, oomRejections: state.oomRejections}, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:resetStats, _from, state),
|
||||||
|
do: {:reply, :ok, %{state | evictedKeys: 0, oomRejections: 0}}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_info(:tick, state) do
|
||||||
|
state = maintain(state)
|
||||||
|
schedule()
|
||||||
|
{:noreply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp maintain(state) do
|
||||||
|
cond do
|
||||||
|
Config.maxmemory() > 0 ->
|
||||||
|
elem(sweep(state), 1)
|
||||||
|
|
||||||
|
Config.tracksAccess?() ->
|
||||||
|
pruneMetadata(Store.dbsize(), fn -> Store.keys("*") end)
|
||||||
|
state
|
||||||
|
|
||||||
|
true ->
|
||||||
|
state
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp decide(limit) do
|
||||||
|
meter = meter()
|
||||||
|
checkedAt = :counters.get(meter, @checkedAt)
|
||||||
|
estimated = :counters.get(meter, @used) + :counters.get(meter, @dirty)
|
||||||
|
|
||||||
|
cond do
|
||||||
|
checkedAt == 0 -> enforce()
|
||||||
|
estimated <= limit -> :ok
|
||||||
|
recent?(checkedAt) -> cached(meter)
|
||||||
|
true -> enforce()
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp recent?(checkedAt) do
|
||||||
|
window = Application.get_env(:excache, :eviction_recheck, @defaultRecheck)
|
||||||
|
window > 0 and System.system_time(:millisecond) - checkedAt < window
|
||||||
|
end
|
||||||
|
|
||||||
|
defp cached(meter), do: if(:counters.get(meter, @verdict) == 0, do: :ok, else: :oom)
|
||||||
|
|
||||||
|
defp sweep(state) do
|
||||||
|
entries = Store.dump()
|
||||||
|
pruneMetadata(length(entries), fn -> Enum.map(entries, &elem(&1, 0)) end)
|
||||||
|
|
||||||
|
used = Memory.usedMemory(entries)
|
||||||
|
limit = Config.maxmemory()
|
||||||
|
policy = Config.policy()
|
||||||
|
|
||||||
|
cond do
|
||||||
|
limit <= 0 or used <= limit ->
|
||||||
|
record(used, :ok)
|
||||||
|
{:ok, state}
|
||||||
|
|
||||||
|
policy == :noeviction ->
|
||||||
|
record(used, :oom)
|
||||||
|
{:oom, %{state | oomRejections: state.oomRejections + 1}}
|
||||||
|
|
||||||
|
true ->
|
||||||
|
evict(entries, policy, limit, state)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp evict(entries, policy, limit, state) do
|
||||||
|
{status, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: policy,
|
||||||
|
maxmemory: limit,
|
||||||
|
sampleSize: Config.maxmemorySamples(),
|
||||||
|
tracker: Tracker,
|
||||||
|
keyspace: keyspace(entries)
|
||||||
|
)
|
||||||
|
|
||||||
|
verdict = if status == :ok, do: :ok, else: :oom
|
||||||
|
record(report.usedMemory, verdict)
|
||||||
|
|
||||||
|
state = %{
|
||||||
|
state
|
||||||
|
| evictedKeys: state.evictedKeys + report.evictedCount,
|
||||||
|
oomRejections: state.oomRejections + if(verdict == :oom, do: 1, else: 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
{verdict, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp keyspace(entries) do
|
||||||
|
%{
|
||||||
|
entries: fn -> entries end,
|
||||||
|
delete: fn key ->
|
||||||
|
deleted = Store.del([key])
|
||||||
|
Aof.append("DEL", [key])
|
||||||
|
Notify.key_event(key, "evicted")
|
||||||
|
deleted
|
||||||
|
end
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp pruneMetadata(keyCount, keysFun) do
|
||||||
|
if Config.tracksAccess?() and Tracker.size(Tracker) > keyCount do
|
||||||
|
Tracker.retain(Tracker, keysFun.())
|
||||||
|
end
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
defp restartTracker do
|
||||||
|
Supervisor.terminate_child(Excache.Supervisor, Tracker)
|
||||||
|
Supervisor.delete_child(Excache.Supervisor, Tracker)
|
||||||
|
Supervisor.start_child(Excache.Supervisor, Excache.Application.trackerSpec())
|
||||||
|
end
|
||||||
|
|
||||||
|
defp record(used, verdict) do
|
||||||
|
meter = meter()
|
||||||
|
:counters.put(meter, @used, used)
|
||||||
|
:counters.put(meter, @dirty, 0)
|
||||||
|
:counters.put(meter, @checkedAt, System.system_time(:millisecond))
|
||||||
|
:counters.put(meter, @verdict, if(verdict == :ok, do: 0, else: 1))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp argBytes(args),
|
||||||
|
do: Enum.reduce(args, Memory.entryOverhead(), fn arg, total -> total + byte_size(arg) end)
|
||||||
|
|
||||||
|
defp meter, do: :persistent_term.get(@meter)
|
||||||
|
|
||||||
|
defp schedule do
|
||||||
|
interval = Application.get_env(:excache, :eviction_interval, @defaultInterval)
|
||||||
|
Process.send_after(self(), :tick, interval)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
defmodule Excache.Eviction.Tracker do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
# Per key access metadata for the LRU and LFU policies.
|
||||||
|
#
|
||||||
|
# The LFU counter follows Redis evict.c: it starts at LFU_INIT_VAL, grows
|
||||||
|
# logarithmically so a key read a million times cannot bury one read a
|
||||||
|
# thousand times, and decays by one unit for every lfu-decay-time minutes a
|
||||||
|
# key goes untouched. Both knobs and the random source are injectable so the
|
||||||
|
# behaviour is reproducible in tests; every timestamp is epoch milliseconds
|
||||||
|
# supplied by the caller, the tracker never reads the clock itself.
|
||||||
|
|
||||||
|
use GenServer
|
||||||
|
|
||||||
|
@lfuInitCounter 5
|
||||||
|
@lfuMaxCounter 255
|
||||||
|
@defaultLogFactor 10
|
||||||
|
@defaultDecayTime 1
|
||||||
|
@defaultSeed {1721, 4093, 8677}
|
||||||
|
|
||||||
|
def lfuInitCounter, do: @lfuInitCounter
|
||||||
|
def lfuMaxCounter, do: @lfuMaxCounter
|
||||||
|
|
||||||
|
def start_link(opts \\ []) do
|
||||||
|
{serverOpts, initOpts} = Keyword.split(opts, [:name])
|
||||||
|
GenServer.start_link(__MODULE__, initOpts, serverOpts)
|
||||||
|
end
|
||||||
|
|
||||||
|
def touch(server, key, now), do: GenServer.call(server, {:touch, key, now})
|
||||||
|
|
||||||
|
def touchAsync(server, key, now), do: GenServer.cast(server, {:touch, key, now})
|
||||||
|
|
||||||
|
def touchMany(server, keys, now), do: GenServer.call(server, {:touchMany, keys, now})
|
||||||
|
|
||||||
|
def forget(server, key), do: GenServer.call(server, {:forget, key})
|
||||||
|
|
||||||
|
def retain(server, keys), do: GenServer.call(server, {:retain, keys})
|
||||||
|
|
||||||
|
def clear(server), do: GenServer.call(server, :clear)
|
||||||
|
|
||||||
|
def size(server), do: GenServer.call(server, :size)
|
||||||
|
|
||||||
|
def lookup(server, key, now), do: GenServer.call(server, {:lookup, key, now})
|
||||||
|
|
||||||
|
def metadata(server, keys, now), do: GenServer.call(server, {:metadata, keys, now})
|
||||||
|
|
||||||
|
def snapshot(server, now), do: GenServer.call(server, {:snapshot, now})
|
||||||
|
|
||||||
|
def counter(server, key, now) do
|
||||||
|
case lookup(server, key, now) do
|
||||||
|
nil -> 0
|
||||||
|
meta -> meta.counter
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def idleMs(server, key, now) do
|
||||||
|
case lookup(server, key, now) do
|
||||||
|
nil -> nil
|
||||||
|
meta -> meta.idleMs
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def settings(server), do: GenServer.call(server, :settings)
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def init(opts) do
|
||||||
|
{:ok,
|
||||||
|
%{
|
||||||
|
access: %{},
|
||||||
|
logFactor: Keyword.get(opts, :logFactor, @defaultLogFactor),
|
||||||
|
decayTime: Keyword.get(opts, :decayTime, @defaultDecayTime),
|
||||||
|
randState: :rand.seed_s(:exsss, Keyword.get(opts, :seed, @defaultSeed))
|
||||||
|
}}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:touch, key, now}, _from, state) do
|
||||||
|
{counter, state} = register(state, key, now)
|
||||||
|
{:reply, counter, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:touchMany, keys, now}, _from, state) do
|
||||||
|
{counters, state} =
|
||||||
|
Enum.reduce(keys, {%{}, state}, fn key, {acc, current} ->
|
||||||
|
{counter, next} = register(current, key, now)
|
||||||
|
{Map.put(acc, key, counter), next}
|
||||||
|
end)
|
||||||
|
|
||||||
|
{:reply, counters, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:forget, key}, _from, state),
|
||||||
|
do: {:reply, :ok, %{state | access: Map.delete(state.access, key)}}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:retain, keys}, _from, state),
|
||||||
|
do: {:reply, :ok, %{state | access: Map.take(state.access, keys)}}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:clear, _from, state), do: {:reply, :ok, %{state | access: %{}}}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:size, _from, state), do: {:reply, map_size(state.access), state}
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:lookup, key, now}, _from, state) do
|
||||||
|
reply =
|
||||||
|
case Map.fetch(state.access, key) do
|
||||||
|
:error -> nil
|
||||||
|
{:ok, meta} -> present(meta, now, state.decayTime)
|
||||||
|
end
|
||||||
|
|
||||||
|
{:reply, reply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:metadata, keys, now}, _from, state) do
|
||||||
|
reply =
|
||||||
|
state.access
|
||||||
|
|> Map.take(keys)
|
||||||
|
|> Map.new(fn {key, meta} -> {key, present(meta, now, state.decayTime)} end)
|
||||||
|
|
||||||
|
{:reply, reply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call({:snapshot, now}, _from, state) do
|
||||||
|
reply =
|
||||||
|
Map.new(state.access, fn {key, meta} -> {key, present(meta, now, state.decayTime)} end)
|
||||||
|
|
||||||
|
{:reply, reply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_call(:settings, _from, state) do
|
||||||
|
{:reply, %{logFactor: state.logFactor, decayTime: state.decayTime}, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def handle_cast({:touch, key, now}, state) do
|
||||||
|
{_counter, state} = register(state, key, now)
|
||||||
|
{:noreply, state}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp register(state, key, now) do
|
||||||
|
{counter, randState} =
|
||||||
|
case Map.fetch(state.access, key) do
|
||||||
|
:error ->
|
||||||
|
{@lfuInitCounter, state.randState}
|
||||||
|
|
||||||
|
{:ok, meta} ->
|
||||||
|
meta.counter
|
||||||
|
|> decay(meta.lastAccess, now, state.decayTime)
|
||||||
|
|> logIncrement(state.logFactor, state.randState)
|
||||||
|
end
|
||||||
|
|
||||||
|
access = Map.put(state.access, key, %{lastAccess: now, counter: counter})
|
||||||
|
{counter, %{state | access: access, randState: randState}}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp logIncrement(@lfuMaxCounter, _logFactor, randState), do: {@lfuMaxCounter, randState}
|
||||||
|
|
||||||
|
defp logIncrement(counter, logFactor, randState) do
|
||||||
|
{roll, nextState} = :rand.uniform_s(randState)
|
||||||
|
baseval = max(counter - @lfuInitCounter, 0)
|
||||||
|
probability = 1.0 / (baseval * logFactor + 1)
|
||||||
|
|
||||||
|
if roll < probability, do: {counter + 1, nextState}, else: {counter, nextState}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp decay(counter, _lastAccess, _now, decayTime) when decayTime <= 0, do: counter
|
||||||
|
|
||||||
|
defp decay(counter, lastAccess, now, decayTime) do
|
||||||
|
periods = div(div(max(now - lastAccess, 0), 60_000), decayTime)
|
||||||
|
max(counter - periods, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp present(meta, now, decayTime) do
|
||||||
|
%{
|
||||||
|
lastAccess: meta.lastAccess,
|
||||||
|
idleMs: max(now - meta.lastAccess, 0),
|
||||||
|
counter: decay(meta.counter, meta.lastAccess, now, decayTime)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
defmodule Excache.Memory do
|
||||||
|
@moduledoc false
|
||||||
|
|
||||||
|
# Byte accounting for stored entries, the input every eviction policy needs.
|
||||||
|
#
|
||||||
|
# Structural cost comes from :erts_debug.size/1, which walks the term and
|
||||||
|
# counts machine words (shared subterms once). Two things it cannot see are
|
||||||
|
# added explicitly:
|
||||||
|
#
|
||||||
|
# * reference counted binaries. ERTS keeps binaries up to 64 bytes on the
|
||||||
|
# heap, so :erts_debug.size/1 covers those; anything larger lives
|
||||||
|
# off-heap and only its ProcBin header is counted, so the payload is
|
||||||
|
# added by walking the term.
|
||||||
|
#
|
||||||
|
# * the per-row cost of living in an ETS set. Measured against
|
||||||
|
# :ets.info(table, :memory) deltas on 64-bit OTP it is 41.24 bytes per
|
||||||
|
# row above the flat term size: five words of hash bucket bookkeeping
|
||||||
|
# plus an amortised bucket slot. Together with the five words of the
|
||||||
|
# {key, kind, value, expiresAt} tuple itself (header plus four slots)
|
||||||
|
# that is the ten word constant below; memory_test.exs asserts the
|
||||||
|
# resulting estimate against a real ETS insert.
|
||||||
|
|
||||||
|
alias Excache.SkipList
|
||||||
|
alias Excache.Store
|
||||||
|
alias Excache.Stream, as: CacheStream
|
||||||
|
|
||||||
|
@kinds [:string, :list, :hash, :set, :zset, :stream]
|
||||||
|
@entryOverheadWords 10
|
||||||
|
@heapBinaryLimit 64
|
||||||
|
|
||||||
|
def kinds, do: @kinds
|
||||||
|
|
||||||
|
def wordSize, do: :erlang.system_info(:wordsize)
|
||||||
|
|
||||||
|
def entryOverhead, do: @entryOverheadWords * wordSize()
|
||||||
|
|
||||||
|
def keyBytes(key) when is_binary(key), do: termBytes(key)
|
||||||
|
|
||||||
|
def estimateEntry(key, {kind, value, _expiresAt}) do
|
||||||
|
keyBytes(key) + estimateValue(kind, value) + entryOverhead()
|
||||||
|
end
|
||||||
|
|
||||||
|
def entryBytes({key, kind, value, expiresAt}),
|
||||||
|
do: estimateEntry(key, {kind, value, expiresAt})
|
||||||
|
|
||||||
|
def estimateValue(:string, value) when is_binary(value), do: termBytes(value)
|
||||||
|
def estimateValue(:list, value) when is_list(value), do: termBytes(value)
|
||||||
|
def estimateValue(:set, %MapSet{} = value), do: termBytes(value)
|
||||||
|
def estimateValue(:zset, %SkipList{} = value), do: termBytes(value)
|
||||||
|
def estimateValue(:stream, %CacheStream{} = value), do: termBytes(value)
|
||||||
|
def estimateValue(:hash, value) when is_map(value), do: termBytes(value)
|
||||||
|
|
||||||
|
def elementCount(:string, value) when is_binary(value), do: 1
|
||||||
|
def elementCount(:list, value), do: length(value)
|
||||||
|
def elementCount(:set, value), do: MapSet.size(value)
|
||||||
|
def elementCount(:zset, value), do: SkipList.size(value)
|
||||||
|
def elementCount(:stream, value), do: CacheStream.length(value)
|
||||||
|
def elementCount(:hash, value) when is_map(value), do: map_size(value)
|
||||||
|
|
||||||
|
def payloadBytes(:string, value), do: byte_size(value)
|
||||||
|
|
||||||
|
def payloadBytes(:list, value), do: sumBytes(value)
|
||||||
|
|
||||||
|
def payloadBytes(:set, value), do: value |> MapSet.to_list() |> sumBytes()
|
||||||
|
|
||||||
|
def payloadBytes(:zset, value) do
|
||||||
|
Enum.reduce(SkipList.to_list(value), 0, fn {_score, member}, total ->
|
||||||
|
total + byte_size(member) + wordSize()
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def payloadBytes(:stream, value) do
|
||||||
|
Enum.reduce(CacheStream.toList(value), 0, fn {_id, fields}, total ->
|
||||||
|
total + 2 * wordSize() + sumBytes(fields)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def payloadBytes(:hash, value) when is_map(value) do
|
||||||
|
Enum.reduce(value, 0, fn {field, member}, total ->
|
||||||
|
total + byte_size(field) + byte_size(member)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def describe(key, {kind, value, expiresAt}) do
|
||||||
|
total = estimateEntry(key, {kind, value, expiresAt})
|
||||||
|
payload = payloadBytes(kind, value)
|
||||||
|
|
||||||
|
%{
|
||||||
|
key: key,
|
||||||
|
kind: kind,
|
||||||
|
bytes: total,
|
||||||
|
keyBytes: keyBytes(key),
|
||||||
|
valueBytes: estimateValue(kind, value),
|
||||||
|
payloadBytes: payload,
|
||||||
|
overheadBytes: total - payload,
|
||||||
|
elements: elementCount(kind, value),
|
||||||
|
volatile: expiresAt != nil
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def usage(key) do
|
||||||
|
case Store.watch_snapshot(key) do
|
||||||
|
:miss -> nil
|
||||||
|
entry -> estimateEntry(key, entry)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def describeKey(key) do
|
||||||
|
case Store.watch_snapshot(key) do
|
||||||
|
:miss -> nil
|
||||||
|
entry -> describe(key, entry)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def usedMemory, do: usedMemory(Store.dump())
|
||||||
|
|
||||||
|
def usedMemory(entries) when is_list(entries),
|
||||||
|
do: Enum.reduce(entries, 0, fn entry, total -> total + entryBytes(entry) end)
|
||||||
|
|
||||||
|
def usageByType, do: usageByType(Store.dump())
|
||||||
|
|
||||||
|
def usageByType(entries) when is_list(entries) do
|
||||||
|
empty = Map.new(@kinds, &{&1, %{keys: 0, bytes: 0}})
|
||||||
|
|
||||||
|
Enum.reduce(entries, empty, fn {_key, kind, _value, _expiresAt} = entry, acc ->
|
||||||
|
Map.update!(acc, kind, fn %{keys: keys, bytes: bytes} ->
|
||||||
|
%{keys: keys + 1, bytes: bytes + entryBytes(entry)}
|
||||||
|
end)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def stats, do: stats(Store.dump())
|
||||||
|
|
||||||
|
def stats(entries) when is_list(entries) do
|
||||||
|
keys = length(entries)
|
||||||
|
used = usedMemory(entries)
|
||||||
|
|
||||||
|
overhead =
|
||||||
|
Enum.reduce(entries, keys * entryOverhead(), fn {key, _kind, _value, _expiresAt}, total ->
|
||||||
|
total + keyBytes(key)
|
||||||
|
end)
|
||||||
|
|
||||||
|
%{
|
||||||
|
keys: keys,
|
||||||
|
usedMemory: used,
|
||||||
|
overheadBytes: overhead,
|
||||||
|
datasetBytes: used - overhead,
|
||||||
|
bytesPerKey: if(keys == 0, do: 0, else: div(used, keys)),
|
||||||
|
volatileKeys:
|
||||||
|
Enum.count(entries, fn {_key, _kind, _value, expiresAt} -> expiresAt != nil end),
|
||||||
|
byType: usageByType(entries),
|
||||||
|
largestKey: largestKey(entries)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp largestKey([]), do: nil
|
||||||
|
|
||||||
|
defp largestKey(entries) do
|
||||||
|
entries
|
||||||
|
|> Enum.map(fn {key, _kind, _value, _expiresAt} = entry -> {key, entryBytes(entry)} end)
|
||||||
|
|> Enum.max_by(fn {_key, bytes} -> bytes end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp termBytes(term), do: :erts_debug.size(term) * wordSize() + refcBytes(term)
|
||||||
|
|
||||||
|
defp sumBytes(binaries),
|
||||||
|
do: Enum.reduce(binaries, 0, fn binary, total -> total + byte_size(binary) end)
|
||||||
|
|
||||||
|
defp refcBytes(term) when is_binary(term) do
|
||||||
|
size = byte_size(term)
|
||||||
|
if size > @heapBinaryLimit, do: size, else: 0
|
||||||
|
end
|
||||||
|
|
||||||
|
defp refcBytes(term) when is_list(term),
|
||||||
|
do: Enum.reduce(term, 0, fn element, total -> total + refcBytes(element) end)
|
||||||
|
|
||||||
|
defp refcBytes(%_struct{} = term), do: term |> Map.from_struct() |> refcBytes()
|
||||||
|
|
||||||
|
defp refcBytes(term) when is_map(term) do
|
||||||
|
Enum.reduce(term, 0, fn {mapKey, mapValue}, total ->
|
||||||
|
total + refcBytes(mapKey) + refcBytes(mapValue)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp refcBytes(term) when is_tuple(term), do: term |> Tuple.to_list() |> refcBytes()
|
||||||
|
|
||||||
|
defp refcBytes(_term), do: 0
|
||||||
|
end
|
||||||
@@ -11,7 +11,7 @@ defmodule Excache.Server.Connection do
|
|||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
def init(socket) do
|
def init(socket) do
|
||||||
base = %{socket: socket, buffer: "", subscriptions: MapSet.new()}
|
base = %{socket: socket, buffer: "", subscriptions: MapSet.new(), blocked: nil}
|
||||||
{:ok, Map.merge(base, Transaction.initial_state())}
|
{:ok, Map.merge(base, Transaction.initial_state())}
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -22,16 +22,16 @@ defmodule Excache.Server.Connection do
|
|||||||
end
|
end
|
||||||
|
|
||||||
def handle_info({:tcp, socket, data}, state) do
|
def handle_info({:tcp, socket, data}, state) do
|
||||||
{output, state} = consume(state.buffer <> data, %{state | buffer: ""})
|
state = pump(%{state | buffer: state.buffer <> data})
|
||||||
|
|
||||||
unless output == [] do
|
|
||||||
:gen_tcp.send(socket, output)
|
|
||||||
end
|
|
||||||
|
|
||||||
:inet.setopts(socket, active: :once)
|
:inet.setopts(socket, active: :once)
|
||||||
{:noreply, state}
|
{:noreply, state}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def handle_info({:excacheUnblocked, ref, reply}, %{blocked: ref} = state) do
|
||||||
|
:gen_tcp.send(state.socket, Encoder.encode(reply))
|
||||||
|
{:noreply, pump(%{state | blocked: nil})}
|
||||||
|
end
|
||||||
|
|
||||||
def handle_info({:tcp_closed, _socket}, state), do: {:stop, :normal, state}
|
def handle_info({:tcp_closed, _socket}, state), do: {:stop, :normal, state}
|
||||||
def handle_info({:tcp_error, _socket, _reason}, state), do: {:stop, :normal, state}
|
def handle_info({:tcp_error, _socket, _reason}, state), do: {:stop, :normal, state}
|
||||||
|
|
||||||
@@ -41,6 +41,14 @@ defmodule Excache.Server.Connection do
|
|||||||
{:noreply, state}
|
{:noreply, state}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp pump(%{blocked: nil} = state) do
|
||||||
|
{output, state} = consume(state.buffer, %{state | buffer: ""})
|
||||||
|
unless output == [], do: :gen_tcp.send(state.socket, output)
|
||||||
|
state
|
||||||
|
end
|
||||||
|
|
||||||
|
defp pump(state), do: state
|
||||||
|
|
||||||
defp consume(buffer, state) do
|
defp consume(buffer, state) do
|
||||||
case Parser.parse_command(buffer) do
|
case Parser.parse_command(buffer) do
|
||||||
:incomplete ->
|
:incomplete ->
|
||||||
@@ -51,9 +59,16 @@ defmodule Excache.Server.Connection do
|
|||||||
|
|
||||||
{:ok, args, rest} ->
|
{:ok, args, rest} ->
|
||||||
{replies, state} = Command.execute(args, state)
|
{replies, state} = Command.execute(args, state)
|
||||||
encoded = Enum.map(replies, &Encoder.encode/1)
|
continue(replies, rest, state)
|
||||||
{more, state} = consume(rest, state)
|
|
||||||
{[encoded | more], state}
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp continue([{:blocked, ref}], rest, state),
|
||||||
|
do: {[], %{state | buffer: rest, blocked: ref}}
|
||||||
|
|
||||||
|
defp continue(replies, rest, state) do
|
||||||
|
encoded = Enum.map(replies, &Encoder.encode/1)
|
||||||
|
{more, state} = consume(rest, state)
|
||||||
|
{[encoded | more], state}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
+102
-34
@@ -2,6 +2,8 @@ defmodule Excache.Store do
|
|||||||
@moduledoc false
|
@moduledoc false
|
||||||
|
|
||||||
alias Excache.{Bitmap, HyperLogLog}
|
alias Excache.{Bitmap, HyperLogLog}
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
|
||||||
def shard_count, do: Application.get_env(:excache, :shard_count, 8)
|
def shard_count, do: Application.get_env(:excache, :shard_count, 8)
|
||||||
|
|
||||||
@@ -9,7 +11,21 @@ defmodule Excache.Store do
|
|||||||
|
|
||||||
defp shard_for(key), do: shard_name(rem(:erlang.phash2(key), shard_count()))
|
defp shard_for(key), do: shard_name(rem(:erlang.phash2(key), shard_count()))
|
||||||
|
|
||||||
defp call_key(key, op), do: GenServer.call(shard_for(key), {:op, op})
|
defp call_key(key, op) do
|
||||||
|
trackAccess(key, op)
|
||||||
|
GenServer.call(shard_for(key), {:op, op})
|
||||||
|
end
|
||||||
|
|
||||||
|
defp trackAccess(_key, {:del, _target}), do: :ok
|
||||||
|
defp trackAccess(_key, {:peek, _target}), do: :ok
|
||||||
|
|
||||||
|
defp trackAccess(key, _op) do
|
||||||
|
if Config.tracksAccess?() do
|
||||||
|
Tracker.touchAsync(Tracker, key, System.system_time(:millisecond))
|
||||||
|
end
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
defp fan_out(op) do
|
defp fan_out(op) do
|
||||||
for index <- 0..(shard_count() - 1) do
|
for index <- 0..(shard_count() - 1) do
|
||||||
@@ -265,23 +281,37 @@ defmodule Excache.Store do
|
|||||||
def touch(keys), do: exists(keys)
|
def touch(keys), do: exists(keys)
|
||||||
|
|
||||||
def hmget(key, fields) do
|
def hmget(key, fields) do
|
||||||
Enum.map(fields, fn field ->
|
collectPerField(fields, fn field ->
|
||||||
case call_key(key, {:hget, key, field}) do
|
case call_key(key, {:hget, key, field}) do
|
||||||
{:ok, value} -> value
|
{:ok, value} -> {:ok, value}
|
||||||
_ -> nil
|
nil -> {:ok, nil}
|
||||||
|
error -> error
|
||||||
end
|
end
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
def smismember(key, members) do
|
def smismember(key, members) do
|
||||||
Enum.map(members, fn member ->
|
collectPerField(members, fn member ->
|
||||||
case call_key(key, {:sismember, key, member}) do
|
case call_key(key, {:sismember, key, member}) do
|
||||||
{:ok, 1} -> 1
|
{:ok, present} -> {:ok, present}
|
||||||
_ -> 0
|
error -> error
|
||||||
end
|
end
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp collectPerField(entries, lookup) do
|
||||||
|
Enum.reduce_while(entries, {:ok, []}, fn entry, {:ok, acc} ->
|
||||||
|
case lookup.(entry) do
|
||||||
|
{:ok, value} -> {:cont, {:ok, [value | acc]}}
|
||||||
|
error -> {:halt, error}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> case do
|
||||||
|
{:ok, values} -> Enum.reverse(values)
|
||||||
|
error -> error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def sintercard(keys, limit) do
|
def sintercard(keys, limit) do
|
||||||
with {:ok, members} <- set_op(keys, &MapSet.intersection/2) do
|
with {:ok, members} <- set_op(keys, &MapSet.intersection/2) do
|
||||||
count = length(members)
|
count = length(members)
|
||||||
@@ -290,30 +320,64 @@ defmodule Excache.Store do
|
|||||||
end
|
end
|
||||||
|
|
||||||
def smove(source, dest, member) do
|
def smove(source, dest, member) do
|
||||||
case call_key(source, {:srem, source, [member]}) do
|
with :ok <- ensureKind(dest, "set") do
|
||||||
{:ok, 0} ->
|
case call_key(source, {:srem, source, [member]}) do
|
||||||
{:ok, 0}
|
{:ok, 0} ->
|
||||||
|
{:ok, 0}
|
||||||
|
|
||||||
{:ok, _} ->
|
{:ok, _removed} ->
|
||||||
call_key(dest, {:sadd, dest, [member]})
|
case call_key(dest, {:sadd, dest, [member]}) do
|
||||||
{:ok, 1}
|
{:ok, _added} ->
|
||||||
|
{:ok, 1}
|
||||||
|
|
||||||
error ->
|
error ->
|
||||||
error
|
call_key(source, {:sadd, source, [member]})
|
||||||
|
error
|
||||||
|
end
|
||||||
|
|
||||||
|
error ->
|
||||||
|
error
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def rpoplpush(source, dest) do
|
def rpoplpush(source, dest) do
|
||||||
case call_key(source, {:rpop, source, nil}) do
|
with :ok <- ensureKind(dest, "list") do
|
||||||
{:ok, value} ->
|
case call_key(source, {:rpop, source, nil}) do
|
||||||
call_key(dest, {:lpush, dest, [value]})
|
{:ok, value} ->
|
||||||
{:ok, value}
|
case call_key(dest, {:lpush, dest, [value]}) do
|
||||||
|
{:ok, _length} ->
|
||||||
|
{:ok, value}
|
||||||
|
|
||||||
nil ->
|
error ->
|
||||||
{:ok, nil}
|
call_key(source, {:rpush, source, [value]})
|
||||||
|
error
|
||||||
|
end
|
||||||
|
|
||||||
error ->
|
nil ->
|
||||||
error
|
{:ok, nil}
|
||||||
|
|
||||||
|
error ->
|
||||||
|
error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp ensureKind(key, expected) do
|
||||||
|
case call_key(key, {:type, key}) do
|
||||||
|
"none" -> :ok
|
||||||
|
^expected -> :ok
|
||||||
|
_ -> {:error, :wrongtype}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp putEntry(dest, {kind, value, ea}),
|
||||||
|
do: call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})
|
||||||
|
|
||||||
|
def rename(key, key) do
|
||||||
|
case call_key(key, {:exists, key}) do
|
||||||
|
1 -> {:ok, "OK"}
|
||||||
|
0 -> {:error, :nokey}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -322,10 +386,13 @@ defmodule Excache.Store do
|
|||||||
:miss ->
|
:miss ->
|
||||||
{:error, :nokey}
|
{:error, :nokey}
|
||||||
|
|
||||||
{kind, value, ea} ->
|
entry ->
|
||||||
call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})
|
if putEntry(dest, entry) == 1 do
|
||||||
call_key(source, {:del, source})
|
call_key(source, {:del, source})
|
||||||
{:ok, "OK"}
|
{:ok, "OK"}
|
||||||
|
else
|
||||||
|
{:error, :nokey}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@@ -340,10 +407,12 @@ defmodule Excache.Store do
|
|||||||
{:ok, 0}
|
{:ok, 0}
|
||||||
|
|
||||||
0 ->
|
0 ->
|
||||||
{kind, value, ea} = entry
|
if putEntry(dest, entry) == 1 do
|
||||||
call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})
|
call_key(source, {:del, source})
|
||||||
call_key(source, {:del, source})
|
{:ok, 1}
|
||||||
{:ok, 1}
|
else
|
||||||
|
{:error, :nokey}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@@ -353,12 +422,11 @@ defmodule Excache.Store do
|
|||||||
:miss ->
|
:miss ->
|
||||||
{:ok, 0}
|
{:ok, 0}
|
||||||
|
|
||||||
{kind, value, ea} ->
|
entry ->
|
||||||
if not replace and call_key(dest, {:exists, dest}) == 1 do
|
if not replace and call_key(dest, {:exists, dest}) == 1 do
|
||||||
{:ok, 0}
|
{:ok, 0}
|
||||||
else
|
else
|
||||||
call_key(dest, {:del, dest})
|
{:ok, putEntry(dest, entry)}
|
||||||
{:ok, call_key(dest, {:extra, {:raw_put, dest, kind, value, ea}})}
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -55,6 +55,9 @@ defmodule Excache.Store.Ops.Binary do
|
|||||||
:wrongtype ->
|
:wrongtype ->
|
||||||
{:error, :wrongtype}
|
{:error, :wrongtype}
|
||||||
|
|
||||||
|
{:ok, current, _ea, _exists} when value == "" ->
|
||||||
|
{:ok, byte_size(current)}
|
||||||
|
|
||||||
{:ok, current, ea, _exists} ->
|
{:ok, current, ea, _exists} ->
|
||||||
updated = overwrite(current, offset, value)
|
updated = overwrite(current, offset, value)
|
||||||
store(table, key, updated, ea)
|
store(table, key, updated, ea)
|
||||||
@@ -149,8 +152,6 @@ defmodule Excache.Store.Ops.Binary do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp overwrite(current, offset, value) when value == "", do: current
|
|
||||||
|
|
||||||
defp overwrite(current, offset, value) do
|
defp overwrite(current, offset, value) do
|
||||||
size = byte_size(current)
|
size = byte_size(current)
|
||||||
padded = if size < offset, do: current <> :binary.copy(<<0>>, offset - size), else: current
|
padded = if size < offset, do: current <> :binary.copy(<<0>>, offset - size), else: current
|
||||||
|
|||||||
@@ -313,22 +313,15 @@ defmodule Excache.Store.Ops.Extra do
|
|||||||
scanned =
|
scanned =
|
||||||
if rank < 0, do: Enum.reverse(indexed), else: indexed
|
if rank < 0, do: Enum.reverse(indexed), else: indexed
|
||||||
|
|
||||||
skip = abs(rank) - 1
|
matches = collectMatches(scanned, element, abs(rank) - 1, matchLimit(count), maxlen, 0, [])
|
||||||
limit = if count == 0, do: :all, else: count
|
|
||||||
|
|
||||||
collectMatches(scanned, element, skip, limit, maxlen, 0, [])
|
if count == nil, do: List.first(matches), else: matches
|
||||||
|> then(fn matches ->
|
|
||||||
if count == nil do
|
|
||||||
case matches do
|
|
||||||
[] -> nil
|
|
||||||
[first | _] -> first
|
|
||||||
end
|
|
||||||
else
|
|
||||||
matches
|
|
||||||
end
|
|
||||||
end)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp matchLimit(nil), do: 1
|
||||||
|
defp matchLimit(0), do: :all
|
||||||
|
defp matchLimit(count), do: count
|
||||||
|
|
||||||
defp collectMatches([], _element, _skip, _limit, _maxlen, _compared, acc),
|
defp collectMatches([], _element, _skip, _limit, _maxlen, _compared, acc),
|
||||||
do: Enum.reverse(acc)
|
do: Enum.reverse(acc)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,444 @@
|
|||||||
|
defmodule Excache.BlockingTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Blocking
|
||||||
|
|
||||||
|
describe "immediate path" do
|
||||||
|
test "BLPOP takes the head when the list already holds data", %{conn: conn} do
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "jobs", "first", "second"])
|
||||||
|
assert {["jobs", "first"], conn} = Client.command(conn, ["BLPOP", "jobs", "0"])
|
||||||
|
assert {["second"], conn} = Client.command(conn, ["LRANGE", "jobs", "0", "-1"])
|
||||||
|
assert {["jobs", "second"], conn} = Client.command(conn, ["BLPOP", "jobs", "0"])
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "jobs"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BRPOP takes the tail", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["RPUSH", "tail", "a", "b", "c"])
|
||||||
|
assert {["tail", "c"], _conn} = Client.command(conn, ["BRPOP", "tail", "0"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BLPOP serves the first key given that has data", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "second", "later"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "first", "sooner"])
|
||||||
|
|
||||||
|
assert {["first", "sooner"], conn} =
|
||||||
|
Client.command(conn, ["BLPOP", "first", "second", "0"])
|
||||||
|
|
||||||
|
assert {["second", "later"], _conn} =
|
||||||
|
Client.command(conn, ["BLPOP", "first", "second", "0"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BLPOP on a key holding another type answers WRONGTYPE without blocking", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "notalist", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, _conn} =
|
||||||
|
Client.command(conn, ["BLPOP", "notalist", "0"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "argument errors", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR timeout is negative"}, conn} =
|
||||||
|
Client.command(conn, ["BLPOP", "k", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR timeout is not a float or out of range"}, conn} =
|
||||||
|
Client.command(conn, ["BLPOP", "k", "later"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'blpop' command"}, conn} =
|
||||||
|
Client.command(conn, ["BLPOP", "k"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["BLMOVE", "a", "b", "LEFT", "SIDEWAYS", "0"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "blocking path" do
|
||||||
|
test "BLPOP waits until another connection pushes", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "queue", "0"])
|
||||||
|
awaitWaiters("queue", 1)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "queue", "payload"])
|
||||||
|
assert Task.await(waiter) == ["queue", "payload"]
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "queue"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BRPOP waits on the tail side", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BRPOP", "queue", "0"])
|
||||||
|
awaitWaiters("queue", 1)
|
||||||
|
|
||||||
|
assert {2, _conn} = Client.command(conn, ["RPUSH", "queue", "old", "new"])
|
||||||
|
assert Task.await(waiter) == ["queue", "new"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a waiter registered on several keys wakes on whichever key receives data", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "left", "right", "0"])
|
||||||
|
awaitWaiters("right", 1)
|
||||||
|
|
||||||
|
assert {1, _conn} = Client.command(conn, ["LPUSH", "right", "hello"])
|
||||||
|
assert Task.await(waiter) == ["right", "hello"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a timed out BLPOP answers with a nil array on the wire", %{port: port} do
|
||||||
|
socket = rawConnect(port)
|
||||||
|
:ok = :gen_tcp.send(socket, encodeCommand(["BLPOP", "never", "0.15"]))
|
||||||
|
|
||||||
|
{elapsed, reply} = :timer.tc(fn -> :gen_tcp.recv(socket, 0, 2000) end)
|
||||||
|
|
||||||
|
assert reply == {:ok, "*-1\r\n"}
|
||||||
|
assert elapsed > 100_000
|
||||||
|
assert elapsed < 1_500_000
|
||||||
|
|
||||||
|
:gen_tcp.close(socket)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a blocked connection still answers commands sent after it is unblocked", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
pipelined = Client.connect(port)
|
||||||
|
|
||||||
|
pipelined =
|
||||||
|
Client.send_raw(pipelined, [
|
||||||
|
encodeCommand(["BLPOP", "pipeq", "0"]),
|
||||||
|
encodeCommand(["PING"])
|
||||||
|
])
|
||||||
|
|
||||||
|
awaitWaiters("pipeq", 1)
|
||||||
|
assert :gen_tcp.recv(pipelined.socket, 0, 100) == {:error, :timeout}
|
||||||
|
|
||||||
|
assert {1, _conn} = Client.command(conn, ["LPUSH", "pipeq", "v"])
|
||||||
|
|
||||||
|
assert {["pipeq", "v"], pipelined} = Client.read(pipelined)
|
||||||
|
assert {{:simple, "PONG"}, pipelined} = Client.read(pipelined)
|
||||||
|
|
||||||
|
Client.close(pipelined)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "delivery accounting" do
|
||||||
|
test "three waiters are served in the order they blocked", %{conn: conn, port: port} do
|
||||||
|
first = blockingTask(port, ["BLPOP", "fifo", "0"])
|
||||||
|
awaitWaiters("fifo", 1)
|
||||||
|
second = blockingTask(port, ["BLPOP", "fifo", "0"])
|
||||||
|
awaitWaiters("fifo", 2)
|
||||||
|
third = blockingTask(port, ["BLPOP", "fifo", "0"])
|
||||||
|
awaitWaiters("fifo", 3)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "fifo", "a"])
|
||||||
|
assert Task.await(first) == ["fifo", "a"]
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "fifo", "b"])
|
||||||
|
assert Task.await(second) == ["fifo", "b"]
|
||||||
|
|
||||||
|
assert {1, _conn} = Client.command(conn, ["LPUSH", "fifo", "c"])
|
||||||
|
assert Task.await(third) == ["fifo", "c"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "one pushed element wakes exactly one of two waiters", %{conn: conn, port: port} do
|
||||||
|
first = blockingTask(port, ["BLPOP", "single", "0"])
|
||||||
|
awaitWaiters("single", 1)
|
||||||
|
second = blockingTask(port, ["BLPOP", "single", "0"])
|
||||||
|
awaitWaiters("single", 2)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "single", "only"])
|
||||||
|
assert Task.await(first) == ["single", "only"]
|
||||||
|
|
||||||
|
assert Task.yield(second, 150) == nil
|
||||||
|
assert Blocking.waitingCount("single") == 1
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "single"])
|
||||||
|
|
||||||
|
assert {1, _conn} = Client.command(conn, ["LPUSH", "single", "later"])
|
||||||
|
assert Task.await(second) == ["single", "later"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a two element push wakes both waiters with one element each", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
first = blockingTask(port, ["BLPOP", "pair", "0"])
|
||||||
|
awaitWaiters("pair", 1)
|
||||||
|
second = blockingTask(port, ["BLPOP", "pair", "0"])
|
||||||
|
awaitWaiters("pair", 2)
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["LPUSH", "pair", "a", "b"])
|
||||||
|
|
||||||
|
assert Task.await(first) == ["pair", "b"]
|
||||||
|
assert Task.await(second) == ["pair", "a"]
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "pair"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a dropped connection leaves the element for the next client", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
victim = Client.connect(port)
|
||||||
|
Client.send_raw(victim, encodeCommand(["BLPOP", "orphan", "0"]))
|
||||||
|
awaitWaiters("orphan", 1)
|
||||||
|
|
||||||
|
Client.close(victim)
|
||||||
|
awaitUntil(fn -> Blocking.waitingCount("orphan") == 0 end)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "orphan", "kept"])
|
||||||
|
assert {["kept"], conn} = Client.command(conn, ["LRANGE", "orphan", "0", "-1"])
|
||||||
|
assert {["orphan", "kept"], _conn} = Client.command(conn, ["BLPOP", "orphan", "0"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "isolation" do
|
||||||
|
test "a blocked client stalls neither other clients nor the expiry sweeper", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
subscriber = Client.connect(port)
|
||||||
|
|
||||||
|
assert {["subscribe", "__keyevent@0__:expired", 1], subscriber} =
|
||||||
|
Client.command(subscriber, ["SUBSCRIBE", "__keyevent@0__:expired"])
|
||||||
|
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "idle", "1"])
|
||||||
|
awaitWaiters("idle", 1)
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "other", "v"])
|
||||||
|
assert {"v", conn} = Client.command(conn, ["GET", "other"])
|
||||||
|
assert {3, conn} = Client.command(conn, ["RPUSH", "unrelated", "a", "b", "c"])
|
||||||
|
assert {["a", "b", "c"], conn} = Client.command(conn, ["LRANGE", "unrelated", "0", "-1"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, _conn} =
|
||||||
|
Client.command(conn, ["SET", "ephemeral", "v", "PX", "60"])
|
||||||
|
|
||||||
|
assert {["message", "__keyevent@0__:expired", "ephemeral"], _subscriber} =
|
||||||
|
Client.read(subscriber)
|
||||||
|
|
||||||
|
assert Task.await(waiter) == nil
|
||||||
|
|
||||||
|
Client.close(subscriber)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "blocking commands inside MULTI answer at once instead of waiting", %{port: port} do
|
||||||
|
socket = rawConnect(port)
|
||||||
|
|
||||||
|
:ok =
|
||||||
|
:gen_tcp.send(socket, [
|
||||||
|
encodeCommand(["MULTI"]),
|
||||||
|
encodeCommand(["BLPOP", "txn", "0"]),
|
||||||
|
encodeCommand(["EXEC"])
|
||||||
|
])
|
||||||
|
|
||||||
|
assert readBytes(socket, byte_size("+OK\r\n+QUEUED\r\n*1\r\n*-1\r\n")) ==
|
||||||
|
"+OK\r\n+QUEUED\r\n*1\r\n*-1\r\n"
|
||||||
|
|
||||||
|
:gen_tcp.close(socket)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a queued blocking command still pops available data on EXEC", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "txn", "ready"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
|
||||||
|
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["BLPOP", "txn", "0"])
|
||||||
|
assert {[["txn", "ready"]], conn} = Client.command(conn, ["EXEC"])
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "txn"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a push made inside a transaction wakes a waiter", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "txnpush", "0"])
|
||||||
|
awaitWaiters("txnpush", 1)
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
|
||||||
|
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["LPUSH", "txnpush", "fromtxn"])
|
||||||
|
assert {[1], _conn} = Client.command(conn, ["EXEC"])
|
||||||
|
|
||||||
|
assert Task.await(waiter) == ["txnpush", "fromtxn"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "persistence and notifications" do
|
||||||
|
test "a woken BLPOP is logged as LPOP and replays without blocking", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
aofPath = Application.get_env(:excache, :aof_path)
|
||||||
|
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "aofq", "0"])
|
||||||
|
awaitWaiters("aofq", 1)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "aofq", "payload"])
|
||||||
|
assert Task.await(waiter) == ["aofq", "payload"]
|
||||||
|
|
||||||
|
Excache.Aof.flush()
|
||||||
|
log = File.read!(aofPath)
|
||||||
|
|
||||||
|
assert log =~ "LPOP"
|
||||||
|
refute log =~ "BLPOP"
|
||||||
|
|
||||||
|
Excache.Store.flushdb()
|
||||||
|
Excache.Aof.replay(aofPath)
|
||||||
|
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "aofq"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a woken BLPOP fires the same keyspace event as LPOP", %{conn: conn, port: port} do
|
||||||
|
subscriber = Client.connect(port)
|
||||||
|
|
||||||
|
assert {["subscribe", "__keyevent@0__:lpop", 1], subscriber} =
|
||||||
|
Client.command(subscriber, ["SUBSCRIBE", "__keyevent@0__:lpop"])
|
||||||
|
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "notified", "0"])
|
||||||
|
awaitWaiters("notified", 1)
|
||||||
|
|
||||||
|
assert {1, _conn} = Client.command(conn, ["LPUSH", "notified", "v"])
|
||||||
|
assert Task.await(waiter) == ["notified", "v"]
|
||||||
|
|
||||||
|
assert {["message", "__keyevent@0__:lpop", "notified"], _subscriber} =
|
||||||
|
Client.read(subscriber)
|
||||||
|
|
||||||
|
Client.close(subscriber)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "BLMOVE and BRPOPLPUSH" do
|
||||||
|
test "BRPOPLPUSH moves an available element straight away", %{conn: conn} do
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "src", "a", "b"])
|
||||||
|
assert {"b", conn} = Client.command(conn, ["BRPOPLPUSH", "src", "dst", "0"])
|
||||||
|
assert {["b"], conn} = Client.command(conn, ["LRANGE", "dst", "0", "-1"])
|
||||||
|
assert {["a"], _conn} = Client.command(conn, ["LRANGE", "src", "0", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BRPOPLPUSH waits for the source and times out with nil", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BRPOPLPUSH", "src", "dst", "0"])
|
||||||
|
awaitWaiters("src", 1)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "src", "moved"])
|
||||||
|
assert Task.await(waiter) == "moved"
|
||||||
|
assert {["moved"], conn} = Client.command(conn, ["LRANGE", "dst", "0", "-1"])
|
||||||
|
|
||||||
|
assert {nil, _conn} = Client.command(conn, ["BRPOPLPUSH", "empty", "dst", "0.1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BLMOVE honours both sides while blocked", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BLMOVE", "src", "dst", "LEFT", "RIGHT", "0"])
|
||||||
|
awaitWaiters("src", 1)
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "src", "head", "tail"])
|
||||||
|
assert Task.await(waiter) == "head"
|
||||||
|
|
||||||
|
assert {["tail"], conn} = Client.command(conn, ["LRANGE", "src", "0", "-1"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "dst", "existing"])
|
||||||
|
|
||||||
|
assert {"tail", conn} = Client.command(conn, ["BLMOVE", "src", "dst", "RIGHT", "LEFT", "0"])
|
||||||
|
|
||||||
|
assert {["tail", "head", "existing"], _conn} =
|
||||||
|
Client.command(conn, ["LRANGE", "dst", "0", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a BLMOVE destination wakes a client blocked on it", %{conn: conn, port: port} do
|
||||||
|
consumer = blockingTask(port, ["BLPOP", "stage2", "0"])
|
||||||
|
awaitWaiters("stage2", 1)
|
||||||
|
|
||||||
|
mover = blockingTask(port, ["BLMOVE", "stage1", "stage2", "LEFT", "RIGHT", "0"])
|
||||||
|
awaitWaiters("stage1", 1)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "stage1", "item"])
|
||||||
|
|
||||||
|
assert Task.await(mover) == "item"
|
||||||
|
assert Task.await(consumer) == ["stage2", "item"]
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "stage2"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a rename that replaces a waited key with another type reports WRONGTYPE", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "renamed", "0"])
|
||||||
|
awaitWaiters("renamed", 1)
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", "v"])
|
||||||
|
assert {{:simple, "OK"}, _conn} = Client.command(conn, ["RENAME", "plain", "renamed"])
|
||||||
|
|
||||||
|
assert {:error, "WRONGTYPE" <> _} = Task.await(waiter)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "BLMPOP" do
|
||||||
|
test "pops up to COUNT elements from the first non empty key", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["RPUSH", "mp", "a", "b", "c"])
|
||||||
|
|
||||||
|
assert {["mp", ["a", "b"]], conn} =
|
||||||
|
Client.command(conn, ["BLMPOP", "0", "2", "missing", "mp", "LEFT", "COUNT", "2"])
|
||||||
|
|
||||||
|
assert {["mp", ["c"]], _conn} =
|
||||||
|
Client.command(conn, ["BLMPOP", "0", "1", "mp", "LEFT"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "blocks across several keys and answers RIGHT side", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BLMPOP", "0", "2", "mp1", "mp2", "RIGHT", "COUNT", "2"])
|
||||||
|
awaitWaiters("mp2", 1)
|
||||||
|
|
||||||
|
assert {3, conn} = Client.command(conn, ["RPUSH", "mp2", "x", "y", "z"])
|
||||||
|
assert Task.await(waiter) == ["mp2", ["z", "y"]]
|
||||||
|
assert {["x"], _conn} = Client.command(conn, ["LRANGE", "mp2", "0", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "times out with a nil array", %{conn: conn} do
|
||||||
|
assert {nil, _conn} = Client.command(conn, ["BLMPOP", "0.1", "1", "mp", "LEFT"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "argument errors", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR numkeys should be greater than 0"}, conn} =
|
||||||
|
Client.command(conn, ["BLMPOP", "0", "0", "mp", "LEFT"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["BLMPOP", "0", "1", "mp", "SIDEWAYS"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR count should be greater than 0"}, conn} =
|
||||||
|
Client.command(conn, ["BLMPOP", "0", "1", "mp", "LEFT", "COUNT", "0"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["BLMPOP", "0", "2", "mp", "LEFT"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp blockingTask(port, args) do
|
||||||
|
Task.async(fn ->
|
||||||
|
client = Client.connect(port)
|
||||||
|
{reply, _client} = Client.command(client, args)
|
||||||
|
Client.close(client)
|
||||||
|
reply
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp awaitWaiters(key, count) do
|
||||||
|
awaitUntil(fn -> Blocking.waitingCount(key) == count end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp awaitUntil(check, attempts \\ 400) do
|
||||||
|
cond do
|
||||||
|
check.() ->
|
||||||
|
:ok
|
||||||
|
|
||||||
|
attempts == 0 ->
|
||||||
|
flunk("the expected registry state was never reached")
|
||||||
|
|
||||||
|
true ->
|
||||||
|
Process.sleep(5)
|
||||||
|
awaitUntil(check, attempts - 1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp rawConnect(port) do
|
||||||
|
{:ok, socket} = :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false, packet: :raw])
|
||||||
|
socket
|
||||||
|
end
|
||||||
|
|
||||||
|
defp readBytes(socket, size) do
|
||||||
|
{:ok, bytes} = :gen_tcp.recv(socket, size, 2000)
|
||||||
|
bytes
|
||||||
|
end
|
||||||
|
|
||||||
|
defp encodeCommand(args) do
|
||||||
|
[
|
||||||
|
"*",
|
||||||
|
Integer.to_string(length(args)),
|
||||||
|
"\r\n",
|
||||||
|
Enum.map(args, fn arg -> ["$", Integer.to_string(byte_size(arg)), "\r\n", arg, "\r\n"] end)
|
||||||
|
]
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
defmodule Excache.CommandHandshakeTest do
|
||||||
|
@moduledoc """
|
||||||
|
Client libraries ask the server what it can do before they send anything else.
|
||||||
|
Until the catalog was wired in, `COMMAND` answered with an empty array over the
|
||||||
|
wire - a server that claims to support nothing. These tests pin the answer to
|
||||||
|
the real dispatch table, over a real socket.
|
||||||
|
"""
|
||||||
|
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
test "COMMAND reports the whole dispatch table over the wire", %{conn: conn} do
|
||||||
|
{entries, conn} = Client.command(conn, ["COMMAND"])
|
||||||
|
|
||||||
|
assert is_list(entries)
|
||||||
|
assert length(entries) == length(Enum.uniq(Excache.Command.known_commands()))
|
||||||
|
|
||||||
|
names =
|
||||||
|
entries
|
||||||
|
|> Enum.map(fn [name | _rest] -> String.upcase(name) end)
|
||||||
|
|> MapSet.new()
|
||||||
|
|
||||||
|
for command <- ["GET", "SET", "BLPOP", "MEMORY", "CONFIG", "SCAN", "XADD"] do
|
||||||
|
assert MapSet.member?(names, command), "#{command} missing from COMMAND"
|
||||||
|
end
|
||||||
|
|
||||||
|
{count, _conn} = Client.command(conn, ["COMMAND", "COUNT"])
|
||||||
|
assert count == length(entries)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "COMMAND INFO carries the arity and key positions a router needs", %{conn: conn} do
|
||||||
|
{[["get", arity, flags, firstKey, lastKey, step]], conn} =
|
||||||
|
Client.command(conn, ["COMMAND", "INFO", "GET"])
|
||||||
|
|
||||||
|
assert arity == 2
|
||||||
|
assert firstKey == 1
|
||||||
|
assert lastKey == 1
|
||||||
|
assert step == 1
|
||||||
|
assert is_list(flags)
|
||||||
|
|
||||||
|
{[entry], _conn} = Client.command(conn, ["COMMAND", "INFO", "nosuchcommand"])
|
||||||
|
assert entry == nil
|
||||||
|
end
|
||||||
|
|
||||||
|
test "COMMAND GETKEYS extracts the keys a command touches", %{conn: conn} do
|
||||||
|
assert {["one", "two"], conn} =
|
||||||
|
Client.command(conn, ["COMMAND", "GETKEYS", "MSET", "one", "1", "two", "2"])
|
||||||
|
|
||||||
|
assert {["only"], _conn} = Client.command(conn, ["COMMAND", "GETKEYS", "GET", "only"])
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
defmodule Excache.ConfigTest do
|
||||||
|
use ExUnit.Case, async: false
|
||||||
|
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Config.reset()
|
||||||
|
on_exit(&Config.reset/0)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "maxmemory accepts the Redis unit suffixes and refuses everything else" do
|
||||||
|
sizes = [
|
||||||
|
{"0", 0},
|
||||||
|
{"100", 100},
|
||||||
|
{"512b", 512},
|
||||||
|
{"1k", 1000},
|
||||||
|
{"1kb", 1024},
|
||||||
|
{"1m", 1_000_000},
|
||||||
|
{"8MB", 8 * 1024 * 1024},
|
||||||
|
{"1g", 1_000_000_000},
|
||||||
|
{"2gb", 2 * 1024 * 1024 * 1024}
|
||||||
|
]
|
||||||
|
|
||||||
|
for {raw, bytes} <- sizes do
|
||||||
|
assert Config.set("maxmemory", raw) == :ok
|
||||||
|
assert Config.maxmemory() == bytes
|
||||||
|
end
|
||||||
|
|
||||||
|
for raw <- ["", "abc", "-1", "1 mb", "1tb", "kb", "1.5mb", "1kbb", "mb1"] do
|
||||||
|
assert {:error, {:invalid, hint}} = Config.set("maxmemory", raw)
|
||||||
|
assert hint =~ "memory value"
|
||||||
|
end
|
||||||
|
|
||||||
|
assert Config.maxmemory() == 2 * 1024 * 1024 * 1024
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the default policy is noeviction and every Redis spelling round trips" do
|
||||||
|
assert Config.policy() == :noeviction
|
||||||
|
assert Config.get("maxmemory-policy") == {:ok, :noeviction}
|
||||||
|
|
||||||
|
for policy <- Eviction.policies() do
|
||||||
|
name = Eviction.policyName(policy)
|
||||||
|
assert Config.set("maxmemory-policy", name) == :ok
|
||||||
|
assert Config.policy() == policy
|
||||||
|
assert Config.getGlob("maxmemory-policy") == [{"maxmemory-policy", name}]
|
||||||
|
end
|
||||||
|
|
||||||
|
assert {:error, {:invalid, hint}} = Config.set("maxmemory-policy", "lru")
|
||||||
|
assert hint =~ "allkeys-lru"
|
||||||
|
assert Config.policy() == :volatileRandom
|
||||||
|
end
|
||||||
|
|
||||||
|
test "only the lru and lfu policies switch access tracking on" do
|
||||||
|
refute Config.tracksAccess?()
|
||||||
|
|
||||||
|
for name <- ~w(allkeys-lru allkeys-lfu volatile-lru volatile-lfu) do
|
||||||
|
assert Config.set("maxmemory-policy", name) == :ok
|
||||||
|
assert Config.tracksAccess?()
|
||||||
|
end
|
||||||
|
|
||||||
|
for name <- ~w(noeviction allkeys-random volatile-random volatile-ttl) do
|
||||||
|
assert Config.set("maxmemory-policy", name) == :ok
|
||||||
|
refute Config.tracksAccess?()
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "integer parameters are range checked and keep the last valid value" do
|
||||||
|
assert Config.maxmemorySamples() == Eviction.defaultSampleSize()
|
||||||
|
assert Config.set("maxmemory-samples", "1") == :ok
|
||||||
|
assert Config.set("maxmemory-samples", "64") == :ok
|
||||||
|
assert Config.maxmemorySamples() == 64
|
||||||
|
|
||||||
|
for raw <- ["0", "65", "ten", "-3", ""] do
|
||||||
|
assert {:error, {:invalid, hint}} = Config.set("maxmemory-samples", raw)
|
||||||
|
assert hint == "argument must be an integer between 1 and 64"
|
||||||
|
end
|
||||||
|
|
||||||
|
assert Config.maxmemorySamples() == 64
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the lfu knobs reach the running tracker and come back on reset" do
|
||||||
|
assert Tracker.settings(Tracker) == %{logFactor: 10, decayTime: 1}
|
||||||
|
|
||||||
|
assert Config.set("lfu-log-factor", "42") == :ok
|
||||||
|
assert Config.set("lfu-decay-time", "7") == :ok
|
||||||
|
assert Config.lfuLogFactor() == 42
|
||||||
|
assert Config.lfuDecayTime() == 7
|
||||||
|
assert Tracker.settings(Tracker) == %{logFactor: 42, decayTime: 7}
|
||||||
|
|
||||||
|
Config.reset()
|
||||||
|
assert Tracker.settings(Tracker) == %{logFactor: 10, decayTime: 1}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "port and the aof path are readable but cannot be changed at runtime" do
|
||||||
|
assert Config.get("port") == {:ok, Application.get_env(:excache, :port)}
|
||||||
|
assert Config.set("port", "6380") == {:error, :immutable}
|
||||||
|
|
||||||
|
assert Config.get("aof_path") == {:ok, Excache.Aof.path()}
|
||||||
|
assert Config.set("aof_path", "/tmp/excache-should-not-move.aof") == {:error, :immutable}
|
||||||
|
assert Config.get("aof_path") == {:ok, Excache.Aof.path()}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the snapshot path is settable and persistence follows it" do
|
||||||
|
previous = Application.get_env(:excache, :snapshot_path)
|
||||||
|
on_exit(fn -> Application.put_env(:excache, :snapshot_path, previous) end)
|
||||||
|
|
||||||
|
target = Path.join(System.tmp_dir!(), "excache-config-test.snapshot")
|
||||||
|
|
||||||
|
assert Config.set("snapshot_path", target) == :ok
|
||||||
|
assert Excache.Persistence.snapshot_path() == target
|
||||||
|
assert Config.get("snapshot_path") == {:ok, target}
|
||||||
|
assert {:error, {:invalid, _hint}} = Config.set("snapshot_path", "")
|
||||||
|
assert Excache.Persistence.snapshot_path() == target
|
||||||
|
end
|
||||||
|
|
||||||
|
test "unknown parameters are refused on read, write and glob" do
|
||||||
|
assert Config.get("nosuch") == :error
|
||||||
|
assert Config.set("nosuch", "1") == {:error, :unknown}
|
||||||
|
assert Config.getGlob("nosuch*") == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "getGlob follows glob semantics and answers sorted, formatted pairs" do
|
||||||
|
assert Config.getGlob("maxmemory") == [{"maxmemory", "0"}]
|
||||||
|
|
||||||
|
assert Config.getGlob("maxmemory*") == [
|
||||||
|
{"maxmemory", "0"},
|
||||||
|
{"maxmemory-policy", "noeviction"},
|
||||||
|
{"maxmemory-samples", "5"}
|
||||||
|
]
|
||||||
|
|
||||||
|
assert Config.getGlob("lfu-*") == [{"lfu-decay-time", "1"}, {"lfu-log-factor", "10"}]
|
||||||
|
assert Config.getGlob("?axmemory") == [{"maxmemory", "0"}]
|
||||||
|
assert Config.getGlob("*policy") == [{"maxmemory-policy", "noeviction"}]
|
||||||
|
assert Config.getGlob("MAXMEMORY") == [{"maxmemory", "0"}]
|
||||||
|
assert Config.getGlob("maxmemor") == []
|
||||||
|
|
||||||
|
assert Enum.map(Config.getGlob("*"), fn {name, _value} -> name end) == Config.names()
|
||||||
|
end
|
||||||
|
|
||||||
|
test "parameter names are case insensitive" do
|
||||||
|
assert Config.set("MaxMemory", "2kb") == :ok
|
||||||
|
assert Config.get("MAXMEMORY") == {:ok, 2048}
|
||||||
|
assert Config.getGlob("maxmemory") == [{"maxmemory", "2048"}]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "setMany applies every pair or leaves all of them alone" do
|
||||||
|
assert Config.setMany([{"maxmemory", "4kb"}, {"maxmemory-policy", "allkeys-lfu"}]) == :ok
|
||||||
|
assert Config.maxmemory() == 4096
|
||||||
|
assert Config.policy() == :allkeysLfu
|
||||||
|
|
||||||
|
assert {:error, "maxmemory-policy", {:invalid, _hint}} =
|
||||||
|
Config.setMany([{"maxmemory", "8kb"}, {"maxmemory-policy", "nope"}])
|
||||||
|
|
||||||
|
assert Config.maxmemory() == 4096
|
||||||
|
assert Config.policy() == :allkeysLfu
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
defmodule Excache.EvictionRuntimeTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@payload String.duplicate("x", 1000)
|
||||||
|
@oom {:error, "OOM command not allowed when used memory > 'maxmemory'"}
|
||||||
|
|
||||||
|
setup_all do
|
||||||
|
Application.put_env(:excache, :eviction_interval, 60_000)
|
||||||
|
Process.sleep(250)
|
||||||
|
|
||||||
|
on_exit(fn -> Application.delete_env(:excache, :eviction_interval) end)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Application.put_env(:excache, :eviction_recheck, 0)
|
||||||
|
Config.reset()
|
||||||
|
Tracker.clear(Tracker)
|
||||||
|
Runner.resetStats()
|
||||||
|
|
||||||
|
on_exit(fn ->
|
||||||
|
Config.reset()
|
||||||
|
Application.delete_env(:excache, :eviction_recheck)
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "allkeys-lru drops the coldest keys when a write crosses maxmemory", %{conn: conn} do
|
||||||
|
conn = configure(conn, ["maxmemory-policy", "allkeys-lru", "maxmemory-samples", "64"])
|
||||||
|
conn = fill(conn, 1..40)
|
||||||
|
|
||||||
|
{before, conn} = stats(conn)
|
||||||
|
assert before["keys.count"] == 40
|
||||||
|
assert before["total.allocated"] > 40_000
|
||||||
|
|
||||||
|
Process.sleep(5)
|
||||||
|
conn = touch(conn, 1..10)
|
||||||
|
|
||||||
|
limit = div(before["total.allocated"], 2)
|
||||||
|
conn = configure(conn, ["maxmemory", Integer.to_string(limit)])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "trigger", @payload])
|
||||||
|
|
||||||
|
{evicted, conn} = stats(conn)
|
||||||
|
freed = evicted["evicted.keys"]
|
||||||
|
|
||||||
|
assert freed >= 15 and freed <= 25
|
||||||
|
assert evicted["keys.count"] == 41 - freed
|
||||||
|
assert evicted["total.allocated"] < before["total.allocated"]
|
||||||
|
assert evicted["total.allocated"] <= limit + 2 * before["keys.bytes-per-key"]
|
||||||
|
assert evicted["oom.rejections"] == 0
|
||||||
|
assert Store.dbsize() == evicted["keys.count"]
|
||||||
|
assert Tracker.size(Tracker) == evicted["keys.count"]
|
||||||
|
|
||||||
|
assert {10, conn} = Client.command(conn, ["EXISTS" | keys(1..10)])
|
||||||
|
assert {cold, conn} = Client.command(conn, ["EXISTS" | keys(11..40)])
|
||||||
|
assert cold == 30 - freed
|
||||||
|
|
||||||
|
Aof.flush()
|
||||||
|
deletes = Aof.path() |> File.read!() |> String.split("$3\r\nDEL\r\n") |> length()
|
||||||
|
assert deletes - 1 == freed
|
||||||
|
|
||||||
|
{doctor, _conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
|
||||||
|
assert doctor =~ "#{freed} keys have been evicted"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "noeviction refuses growing writes but keeps freeing ones", %{conn: conn} do
|
||||||
|
conn = fill(conn, 1..30)
|
||||||
|
{before, conn} = stats(conn)
|
||||||
|
limit = div(before["total.allocated"], 2)
|
||||||
|
conn = configure(conn, ["maxmemory", Integer.to_string(limit)])
|
||||||
|
|
||||||
|
assert {@oom, conn} = Client.command(conn, ["SET", "blocked", @payload])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "blocked"])
|
||||||
|
assert {@oom, conn} = Client.command(conn, ["APPEND", "key:1", "more"])
|
||||||
|
assert {@oom, conn} = Client.command(conn, ["RPUSH", "fresh-list", "value"])
|
||||||
|
|
||||||
|
assert {@payload, conn} = Client.command(conn, ["GET", "key:1"])
|
||||||
|
assert {30, conn} = Client.command(conn, ["DBSIZE"])
|
||||||
|
|
||||||
|
{blocked, conn} = stats(conn)
|
||||||
|
assert blocked["evicted.keys"] == 0
|
||||||
|
assert blocked["oom.rejections"] >= 3
|
||||||
|
assert blocked["total.allocated"] == before["total.allocated"]
|
||||||
|
|
||||||
|
assert {20, conn} = Client.command(conn, ["DEL" | keys(1..20)])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "allowed", @payload])
|
||||||
|
assert {11, conn} = Client.command(conn, ["DBSIZE"])
|
||||||
|
|
||||||
|
{recovered, _conn} = stats(conn)
|
||||||
|
assert recovered["total.allocated"] < limit
|
||||||
|
end
|
||||||
|
|
||||||
|
test "maxmemory 0 lifts the limit again", %{conn: conn} do
|
||||||
|
conn = fill(conn, 1..20)
|
||||||
|
{before, conn} = stats(conn)
|
||||||
|
conn = configure(conn, ["maxmemory", Integer.to_string(div(before["total.allocated"], 2))])
|
||||||
|
|
||||||
|
assert {@oom, conn} = Client.command(conn, ["SET", "blocked", @payload])
|
||||||
|
|
||||||
|
conn = configure(conn, ["maxmemory", "0"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "blocked", @payload])
|
||||||
|
assert {21, _conn} = Client.command(conn, ["DBSIZE"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a volatile policy with no TTL anywhere cannot free a thing", %{conn: conn} do
|
||||||
|
conn = configure(conn, ["maxmemory-policy", "volatile-lru", "maxmemory-samples", "64"])
|
||||||
|
conn = fill(conn, 1..20)
|
||||||
|
|
||||||
|
{before, conn} = stats(conn)
|
||||||
|
limit = div(before["total.allocated"], 2)
|
||||||
|
conn = configure(conn, ["maxmemory", Integer.to_string(limit)])
|
||||||
|
|
||||||
|
assert {@oom, conn} = Client.command(conn, ["SET", "blocked", @payload])
|
||||||
|
assert {20, conn} = Client.command(conn, ["DBSIZE"])
|
||||||
|
|
||||||
|
conn =
|
||||||
|
Enum.reduce(1..15, conn, fn index, acc ->
|
||||||
|
{1, acc} = Client.command(acc, ["EXPIRE", "key:#{index}", "600"])
|
||||||
|
acc
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "allowed", @payload])
|
||||||
|
|
||||||
|
{after_, conn} = stats(conn)
|
||||||
|
assert after_["evicted.keys"] >= 10
|
||||||
|
assert after_["keys.count"] == 21 - after_["evicted.keys"]
|
||||||
|
assert {5, _conn} = Client.command(conn, ["EXISTS" | keys(16..20)])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a read refreshes access metadata only while a tracking policy is set", %{conn: conn} do
|
||||||
|
conn = fill(conn, 1..2)
|
||||||
|
assert Tracker.size(Tracker) == 0
|
||||||
|
|
||||||
|
conn = configure(conn, ["maxmemory-policy", "allkeys-lru"])
|
||||||
|
conn = fill(conn, 1..2)
|
||||||
|
assert Tracker.size(Tracker) == 2
|
||||||
|
|
||||||
|
Process.sleep(5)
|
||||||
|
conn = touch(conn, 1..1)
|
||||||
|
|
||||||
|
now = System.system_time(:millisecond)
|
||||||
|
assert Tracker.idleMs(Tracker, "key:1", now) < Tracker.idleMs(Tracker, "key:2", now)
|
||||||
|
|
||||||
|
conn = configure(conn, ["maxmemory-policy", "noeviction"])
|
||||||
|
Tracker.clear(Tracker)
|
||||||
|
_conn = touch(conn, 1..2)
|
||||||
|
assert Tracker.size(Tracker) == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a sweep drops metadata for keys that are gone", %{conn: conn} do
|
||||||
|
conn = configure(conn, ["maxmemory-policy", "allkeys-lru"])
|
||||||
|
conn = fill(conn, 1..10)
|
||||||
|
assert Tracker.size(Tracker) == 10
|
||||||
|
|
||||||
|
assert {5, conn} = Client.command(conn, ["DEL" | keys(1..5)])
|
||||||
|
assert Tracker.size(Tracker) == 10
|
||||||
|
|
||||||
|
conn = configure(conn, ["maxmemory", "1gb"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "fresh", "value"])
|
||||||
|
|
||||||
|
assert Store.dbsize() == 6
|
||||||
|
assert Tracker.size(Tracker) == 6
|
||||||
|
{reported, _conn} = stats(conn)
|
||||||
|
assert reported["tracker.keys"] == 6
|
||||||
|
assert reported["evicted.keys"] == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
defp configure(conn, args) do
|
||||||
|
{{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET" | args])
|
||||||
|
conn
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fill(conn, range) do
|
||||||
|
Enum.reduce(range, conn, fn index, acc ->
|
||||||
|
{{:simple, "OK"}, acc} = Client.command(acc, ["SET", "key:#{index}", @payload])
|
||||||
|
acc
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp touch(conn, range) do
|
||||||
|
Enum.reduce(range, conn, fn index, acc ->
|
||||||
|
{@payload, acc} = Client.command(acc, ["GET", "key:#{index}"])
|
||||||
|
acc
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp keys(range), do: Enum.map(range, &"key:#{&1}")
|
||||||
|
|
||||||
|
defp stats(conn) do
|
||||||
|
{flat, conn} = Client.command(conn, ["MEMORY", "STATS"])
|
||||||
|
{flat |> Enum.chunk_every(2) |> Map.new(fn [name, value] -> {name, value} end), conn}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
defmodule Excache.EvictionTest do
|
||||||
|
use ExUnit.Case, async: false
|
||||||
|
|
||||||
|
alias Excache.Eviction
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Memory
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@now 1_000_000
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Store.flushdb()
|
||||||
|
on_exit(&Store.flushdb/0)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "policy names round trip through the Redis spelling" do
|
||||||
|
for policy <- Eviction.policies() do
|
||||||
|
assert {:ok, ^policy} = Eviction.parsePolicy(Eviction.policyName(policy))
|
||||||
|
end
|
||||||
|
|
||||||
|
assert Enum.sort(Enum.map(Eviction.policies(), &Eviction.policyName/1)) ==
|
||||||
|
Enum.sort(~w(
|
||||||
|
noeviction allkeys-lru allkeys-lfu allkeys-random
|
||||||
|
volatile-lru volatile-lfu volatile-ttl volatile-random
|
||||||
|
))
|
||||||
|
|
||||||
|
assert Eviction.parsePolicy("lru") == :error
|
||||||
|
assert Eviction.volatile?(:volatileTtl)
|
||||||
|
assert Eviction.allkeys?(:allkeysLfu)
|
||||||
|
assert Eviction.random?(:volatileRandom)
|
||||||
|
refute Eviction.volatile?(:allkeysLru)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "every policy picks a different victim from the same keyspace" do
|
||||||
|
opts = [now: @now, metadata: mixedMetadata(), sampleSize: 8, sampler: firstSampler()]
|
||||||
|
entries = mixedEntries()
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysLru] ++ opts) == {:ok, "idle-all"}
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysLfu] ++ opts) == {:ok, "cold-all"}
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysRandom] ++ opts) == {:ok, "random-all"}
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :volatileLru] ++ opts) == {:ok, "idle-ttl"}
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :volatileLfu] ++ opts) == {:ok, "cold-ttl"}
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :volatileTtl] ++ opts) == {:ok, "soon-ttl"}
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :volatileRandom] ++ opts) ==
|
||||||
|
{:ok, "random-ttl"}
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :noeviction] ++ opts) ==
|
||||||
|
{:error, :noCandidates}
|
||||||
|
|
||||||
|
victims =
|
||||||
|
for policy <- Eviction.policies() -- [:noeviction] do
|
||||||
|
{:ok, key} = Eviction.selectVictim(entries, [policy: policy] ++ opts)
|
||||||
|
key
|
||||||
|
end
|
||||||
|
|
||||||
|
assert length(Enum.uniq(victims)) == 7
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an untracked key is evicted before any tracked one" do
|
||||||
|
entries = stringEntries(["tracked", "unseen"], "payload")
|
||||||
|
metadata = %{"tracked" => %{lastAccess: @now - 500_000, counter: 5}}
|
||||||
|
opts = [now: @now, metadata: metadata, sampleSize: 8, sampler: firstSampler()]
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysLru] ++ opts) == {:ok, "unseen"}
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysLfu] ++ opts) == {:ok, "unseen"}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "noeviction refuses to free anything and reports OOM" do
|
||||||
|
entries = stringEntries(~w(k1 k2 k3), "payload")
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
assert {:error, report} =
|
||||||
|
Eviction.run(policy: :noeviction, maxmemory: 1, keyspace: keyspace, now: @now)
|
||||||
|
|
||||||
|
assert report.reason == :noeviction
|
||||||
|
assert report.evicted == []
|
||||||
|
assert report.evictedCount == 0
|
||||||
|
assert report.freedBytes == 0
|
||||||
|
assert report.usedMemory == Memory.usedMemory(entries)
|
||||||
|
assert :ets.info(table, :size) == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "volatile policies leave a keyspace without any TTL untouched" do
|
||||||
|
entries = stringEntries(~w(k1 k2 k3), "payload")
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
for policy <- [:volatileLru, :volatileLfu, :volatileTtl, :volatileRandom] do
|
||||||
|
assert {:error, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: policy,
|
||||||
|
maxmemory: 1,
|
||||||
|
keyspace: keyspace,
|
||||||
|
now: @now,
|
||||||
|
metadata: %{}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.reason == :noVolatileKeys
|
||||||
|
assert report.evictedCount == 0
|
||||||
|
assert report.freedBytes == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
assert :ets.info(table, :size) == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "maxmemory of zero means no limit at all" do
|
||||||
|
entries = stringEntries(~w(k1 k2 k3), "payload")
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
assert {:ok, report} =
|
||||||
|
Eviction.run(policy: :allkeysRandom, maxmemory: 0, keyspace: keyspace, now: @now)
|
||||||
|
|
||||||
|
assert report.reason == :withinLimit
|
||||||
|
assert report.evicted == []
|
||||||
|
assert :ets.info(table, :size) == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the loop stops the moment usage drops under the limit" do
|
||||||
|
keys = for index <- 1..10, do: "k#{String.pad_leading("#{index}", 2, "0")}"
|
||||||
|
entries = stringEntries(keys, "payload")
|
||||||
|
perEntry = Memory.entryBytes(hd(entries))
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
assert {:ok, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: :allkeysRandom,
|
||||||
|
maxmemory: 7 * perEntry,
|
||||||
|
keyspace: keyspace,
|
||||||
|
now: @now,
|
||||||
|
seed: {2, 4, 8}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.reason == :evicted
|
||||||
|
assert report.evictedCount == 3
|
||||||
|
assert report.freedBytes == 3 * perEntry
|
||||||
|
assert report.usedMemory == 7 * perEntry
|
||||||
|
assert report.passes == 3
|
||||||
|
assert :ets.info(table, :size) == 7
|
||||||
|
assert Enum.all?(report.evicted, fn key -> :ets.lookup(table, key) == [] end)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the candidate pool carries a strong candidate between passes" do
|
||||||
|
entries = stringEntries(~w(k1 k2 k3 k4), "payload")
|
||||||
|
perEntry = Memory.entryBytes(hd(entries))
|
||||||
|
{_table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
metadata = %{
|
||||||
|
"k1" => %{lastAccess: @now - 90_000, counter: 9},
|
||||||
|
"k2" => %{lastAccess: @now - 80_000, counter: 9},
|
||||||
|
"k3" => %{lastAccess: @now - 20_000, counter: 9},
|
||||||
|
"k4" => %{lastAccess: @now - 30_000, counter: 9}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert {:ok, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: :allkeysLru,
|
||||||
|
maxmemory: 2 * perEntry,
|
||||||
|
keyspace: keyspace,
|
||||||
|
now: @now,
|
||||||
|
metadata: metadata,
|
||||||
|
sampleSize: 2,
|
||||||
|
sampler: windowSampler()
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.evicted == ["k1", "k2"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "sampling scores a window instead of the whole keyspace" do
|
||||||
|
entries = stringEntries(~w(k1 k2 k3 k4), "payload")
|
||||||
|
|
||||||
|
metadata = %{
|
||||||
|
"k1" => %{lastAccess: @now - 1_000, counter: 9},
|
||||||
|
"k2" => %{lastAccess: @now - 2_000, counter: 9},
|
||||||
|
"k3" => %{lastAccess: @now - 3_000, counter: 9},
|
||||||
|
"k4" => %{lastAccess: @now - 900_000, counter: 9}
|
||||||
|
}
|
||||||
|
|
||||||
|
opts = [now: @now, metadata: metadata, sampler: firstSampler()]
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysLru, sampleSize: 1] ++ opts) ==
|
||||||
|
{:ok, "k1"}
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries, [policy: :allkeysLru, sampleSize: 4] ++ opts) ==
|
||||||
|
{:ok, "k4"}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "volatile-ttl evicts in expiry order and ignores keys without a TTL" do
|
||||||
|
entries = [
|
||||||
|
{"far", :string, "payload", @now + 900_000},
|
||||||
|
{"immortal", :string, "payload", nil},
|
||||||
|
{"near", :string, "payload", @now + 1_000},
|
||||||
|
{"mid", :string, "payload", @now + 60_000}
|
||||||
|
]
|
||||||
|
|
||||||
|
perEntry = Memory.entryBytes(hd(entries))
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
assert {:ok, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: :volatileTtl,
|
||||||
|
maxmemory: 2 * perEntry,
|
||||||
|
keyspace: keyspace,
|
||||||
|
now: @now,
|
||||||
|
sampleSize: 8
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.evicted == ["near", "mid"]
|
||||||
|
assert :ets.lookup(table, "immortal") != []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "volatile candidates can run out before the limit is reached" do
|
||||||
|
entries =
|
||||||
|
stringEntries(~w(a1 a2 a3), "payload") ++
|
||||||
|
[
|
||||||
|
{"t1", :string, "payload", @now + 900_000},
|
||||||
|
{"t2", :string, "payload", @now + 900_000}
|
||||||
|
]
|
||||||
|
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
assert {:error, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: :volatileRandom,
|
||||||
|
maxmemory: 1,
|
||||||
|
keyspace: keyspace,
|
||||||
|
now: @now,
|
||||||
|
sampleSize: 8
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.reason == :exhausted
|
||||||
|
assert Enum.sort(report.evicted) == ["t1", "t2"]
|
||||||
|
assert report.usedMemory > 1
|
||||||
|
assert :ets.info(table, :size) == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "allkeys-lfu backed by a live tracker keeps the hot key and forgets the victim" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 0)
|
||||||
|
Enum.each(1..50, fn _index -> Tracker.touch(tracker, "hot", @now) end)
|
||||||
|
Tracker.touch(tracker, "cold", @now)
|
||||||
|
|
||||||
|
entries = stringEntries(["cold", "hot"], "payload")
|
||||||
|
coldBytes = Memory.entryBytes(hd(entries))
|
||||||
|
{table, keyspace} = fixtureKeyspace(entries)
|
||||||
|
|
||||||
|
assert {:ok, report} =
|
||||||
|
Eviction.run(
|
||||||
|
policy: :allkeysLfu,
|
||||||
|
maxmemory: Memory.usedMemory(entries) - coldBytes,
|
||||||
|
keyspace: keyspace,
|
||||||
|
tracker: tracker,
|
||||||
|
now: @now,
|
||||||
|
sampleSize: 8
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.evicted == ["cold"]
|
||||||
|
assert :ets.lookup(table, "hot") != []
|
||||||
|
assert Tracker.lookup(tracker, "cold", @now) == nil
|
||||||
|
assert Tracker.counter(tracker, "hot", @now) == 54
|
||||||
|
end
|
||||||
|
|
||||||
|
test "volatile-lru backed by a live tracker evicts the idle TTL key" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 0)
|
||||||
|
Tracker.touch(tracker, "stale", @now - 300_000)
|
||||||
|
Tracker.touch(tracker, "fresh", @now - 1_000)
|
||||||
|
Tracker.touch(tracker, "idlest", @now - 900_000)
|
||||||
|
|
||||||
|
entries = [
|
||||||
|
{"fresh", :string, "payload", @now + 900_000},
|
||||||
|
{"stale", :string, "payload", @now + 900_000},
|
||||||
|
{"idlest", :string, "payload", nil}
|
||||||
|
]
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries,
|
||||||
|
policy: :volatileLru,
|
||||||
|
tracker: tracker,
|
||||||
|
now: @now,
|
||||||
|
sampleSize: 8
|
||||||
|
) == {:ok, "stale"}
|
||||||
|
|
||||||
|
assert Eviction.selectVictim(entries,
|
||||||
|
policy: :allkeysLru,
|
||||||
|
tracker: tracker,
|
||||||
|
now: @now,
|
||||||
|
sampleSize: 8
|
||||||
|
) == {:ok, "idlest"}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a run against the live store really removes the keys" do
|
||||||
|
for index <- 1..20 do
|
||||||
|
:ok = Store.set("evict:#{index}", String.duplicate("payload", 20), [])
|
||||||
|
end
|
||||||
|
|
||||||
|
used = Memory.usedMemory()
|
||||||
|
target = div(used, 2)
|
||||||
|
|
||||||
|
assert {:ok, report} =
|
||||||
|
Eviction.run(policy: :allkeysRandom, maxmemory: target, seed: {1, 2, 3})
|
||||||
|
|
||||||
|
assert report.reason == :evicted
|
||||||
|
assert report.evictedCount >= 10
|
||||||
|
assert Memory.usedMemory() <= target
|
||||||
|
assert Memory.usedMemory() == report.usedMemory
|
||||||
|
assert Store.dbsize() == 20 - report.evictedCount
|
||||||
|
|
||||||
|
for key <- report.evicted do
|
||||||
|
assert Store.get(key) == nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "candidates carry the estimated size of the entry they stand for" do
|
||||||
|
entries = [
|
||||||
|
{"small", :string, "v", nil},
|
||||||
|
{"large", :string, :binary.copy("z", 4096), @now + 1_000}
|
||||||
|
]
|
||||||
|
|
||||||
|
[small, large] = Eviction.candidates(entries, :allkeysLru)
|
||||||
|
|
||||||
|
assert small.bytes == Memory.entryBytes({"small", :string, "v", nil})
|
||||||
|
assert large.bytes > 4096
|
||||||
|
assert large.expiresAt == @now + 1_000
|
||||||
|
assert Eviction.candidates(entries, :volatileLru) |> Enum.map(& &1.key) == ["large"]
|
||||||
|
assert Eviction.candidates(entries, :noeviction) == []
|
||||||
|
end
|
||||||
|
|
||||||
|
defp mixedEntries do
|
||||||
|
[
|
||||||
|
{"random-all", :string, "payload", nil},
|
||||||
|
{"idle-all", :string, "payload", nil},
|
||||||
|
{"cold-all", :string, "payload", nil},
|
||||||
|
{"random-ttl", :string, "payload", @now + 900_000},
|
||||||
|
{"idle-ttl", :string, "payload", @now + 900_000},
|
||||||
|
{"cold-ttl", :string, "payload", @now + 900_000},
|
||||||
|
{"soon-ttl", :string, "payload", @now + 1_000},
|
||||||
|
{"filler", :string, "payload", nil}
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp mixedMetadata do
|
||||||
|
%{
|
||||||
|
"random-all" => %{lastAccess: @now - 1_000, counter: 200},
|
||||||
|
"idle-all" => %{lastAccess: @now - 100_000, counter: 200},
|
||||||
|
"cold-all" => %{lastAccess: @now - 1_000, counter: 1},
|
||||||
|
"random-ttl" => %{lastAccess: @now - 1_000, counter: 200},
|
||||||
|
"idle-ttl" => %{lastAccess: @now - 90_000, counter: 200},
|
||||||
|
"cold-ttl" => %{lastAccess: @now - 1_000, counter: 2},
|
||||||
|
"soon-ttl" => %{lastAccess: @now - 1_000, counter: 200},
|
||||||
|
"filler" => %{lastAccess: @now - 500, counter: 250}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp stringEntries(keys, value), do: Enum.map(keys, fn key -> {key, :string, value, nil} end)
|
||||||
|
|
||||||
|
defp fixtureKeyspace(entries) do
|
||||||
|
table = :ets.new(:evictionFixture, [:ordered_set, :public])
|
||||||
|
Enum.each(entries, &:ets.insert(table, &1))
|
||||||
|
|
||||||
|
{table,
|
||||||
|
%{
|
||||||
|
entries: fn -> :ets.tab2list(table) end,
|
||||||
|
delete: fn key -> :ets.delete(table, key) end
|
||||||
|
}}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp firstSampler, do: fn pool, count, randState -> {Enum.take(pool, count), randState} end
|
||||||
|
|
||||||
|
defp windowSampler do
|
||||||
|
{:ok, cursor} = Agent.start_link(fn -> 0 end)
|
||||||
|
|
||||||
|
fn pool, count, randState ->
|
||||||
|
offset = Agent.get_and_update(cursor, fn pass -> {pass, pass + 1} end)
|
||||||
|
{pool |> Enum.drop(offset) |> Enum.take(count), randState}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp startTracker(opts) do
|
||||||
|
id = {Tracker, System.unique_integer([:positive])}
|
||||||
|
start_supervised!(Supervisor.child_spec({Tracker, opts}, id: id))
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
defmodule Excache.Eviction.TrackerTest do
|
||||||
|
use ExUnit.Case, async: true
|
||||||
|
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
|
||||||
|
@minute 60_000
|
||||||
|
|
||||||
|
test "a fresh key starts at the Redis init counter with zero idle time" do
|
||||||
|
tracker = startTracker(logFactor: 0)
|
||||||
|
|
||||||
|
assert Tracker.touch(tracker, "alpha", 1_000) == Tracker.lfuInitCounter()
|
||||||
|
assert Tracker.lookup(tracker, "alpha", 1_000) == %{lastAccess: 1_000, idleMs: 0, counter: 5}
|
||||||
|
assert Tracker.lookup(tracker, "ghost", 1_000) == nil
|
||||||
|
assert Tracker.counter(tracker, "ghost", 1_000) == 0
|
||||||
|
assert Tracker.idleMs(tracker, "ghost", 1_000) == nil
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a linear counter separates a key read a hundred times from one read twice" do
|
||||||
|
tracker = startTracker(logFactor: 0)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
touchTimes(tracker, "hot", 100, now)
|
||||||
|
touchTimes(tracker, "cold", 2, now)
|
||||||
|
|
||||||
|
assert Tracker.counter(tracker, "hot", now) == 104
|
||||||
|
assert Tracker.counter(tracker, "cold", now) == 6
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the log factor compresses a hot counter far below its hit count" do
|
||||||
|
tracker = startTracker(logFactor: 10)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
touchTimes(tracker, "hot", 100, now)
|
||||||
|
touchTimes(tracker, "cold", 2, now)
|
||||||
|
|
||||||
|
hot = Tracker.counter(tracker, "hot", now)
|
||||||
|
cold = Tracker.counter(tracker, "cold", now)
|
||||||
|
|
||||||
|
assert cold == 6
|
||||||
|
assert hot > cold
|
||||||
|
assert hot < 30
|
||||||
|
end
|
||||||
|
|
||||||
|
test "decay strips the advantage of a hot key that goes idle" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 1)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
touchTimes(tracker, "hot", 100, now)
|
||||||
|
touchTimes(tracker, "cold", 2, now)
|
||||||
|
|
||||||
|
partial = now + 5 * @minute
|
||||||
|
assert Tracker.counter(tracker, "hot", partial) == 99
|
||||||
|
assert Tracker.counter(tracker, "cold", partial) == 1
|
||||||
|
|
||||||
|
faded = now + 200 * @minute
|
||||||
|
assert Tracker.counter(tracker, "hot", faded) == 0
|
||||||
|
assert Tracker.counter(tracker, "cold", faded) == 0
|
||||||
|
|
||||||
|
Tracker.touch(tracker, "cold", faded)
|
||||||
|
assert Tracker.counter(tracker, "cold", faded) == 1
|
||||||
|
assert Tracker.counter(tracker, "cold", faded) > Tracker.counter(tracker, "hot", faded)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a decay time of zero freezes the counters" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 0)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
touchTimes(tracker, "hot", 100, now)
|
||||||
|
|
||||||
|
assert Tracker.counter(tracker, "hot", now + 5000 * @minute) == 104
|
||||||
|
assert Tracker.settings(tracker) == %{logFactor: 0, decayTime: 0}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the counter saturates at the eight bit ceiling" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 0)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
touchTimes(tracker, "hammered", 400, now)
|
||||||
|
|
||||||
|
assert Tracker.counter(tracker, "hammered", now) == Tracker.lfuMaxCounter()
|
||||||
|
assert Tracker.touch(tracker, "hammered", now) == 255
|
||||||
|
end
|
||||||
|
|
||||||
|
test "idle time is measured against the last access" do
|
||||||
|
tracker = startTracker(logFactor: 0)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
Tracker.touch(tracker, "stale", now)
|
||||||
|
Tracker.touch(tracker, "fresh", now + 30_000)
|
||||||
|
|
||||||
|
assert Tracker.idleMs(tracker, "stale", now + 30_000) == 30_000
|
||||||
|
assert Tracker.idleMs(tracker, "fresh", now + 30_000) == 0
|
||||||
|
assert Tracker.lookup(tracker, "stale", now - 5_000).idleMs == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "forget, retain and clear prune the metadata" do
|
||||||
|
tracker = startTracker(logFactor: 0)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
Tracker.touchMany(tracker, ["a", "b", "c", "d"], now)
|
||||||
|
assert Tracker.size(tracker) == 4
|
||||||
|
|
||||||
|
Tracker.forget(tracker, "a")
|
||||||
|
assert Tracker.size(tracker) == 3
|
||||||
|
assert Tracker.lookup(tracker, "a", now) == nil
|
||||||
|
|
||||||
|
Tracker.retain(tracker, ["b", "c"])
|
||||||
|
assert Tracker.size(tracker) == 2
|
||||||
|
assert Tracker.lookup(tracker, "d", now) == nil
|
||||||
|
|
||||||
|
Tracker.clear(tracker)
|
||||||
|
assert Tracker.size(tracker) == 0
|
||||||
|
assert Tracker.snapshot(tracker, now) == %{}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "metadata answers only for the requested keys and decays to the given time" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 1)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
touchTimes(tracker, "hot", 10, now)
|
||||||
|
Tracker.touch(tracker, "cold", now)
|
||||||
|
Tracker.touch(tracker, "other", now)
|
||||||
|
|
||||||
|
metadata = Tracker.metadata(tracker, ["hot", "cold", "absent"], now + 3 * @minute)
|
||||||
|
|
||||||
|
assert Map.keys(metadata) |> Enum.sort() == ["cold", "hot"]
|
||||||
|
assert metadata["hot"].counter == 11
|
||||||
|
assert metadata["cold"].counter == 2
|
||||||
|
assert metadata["hot"].idleMs == 3 * @minute
|
||||||
|
|
||||||
|
assert Tracker.snapshot(tracker, now) |> Map.keys() |> Enum.sort() == ["cold", "hot", "other"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "touchMany registers each key once and returns the new counters" do
|
||||||
|
tracker = startTracker(logFactor: 0, decayTime: 0)
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
assert Tracker.touchMany(tracker, ["a", "b"], now) == %{"a" => 5, "b" => 5}
|
||||||
|
assert Tracker.touchMany(tracker, ["a", "a"], now) == %{"a" => 7}
|
||||||
|
assert Tracker.counter(tracker, "a", now) == 7
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the same seed replays the same logarithmic counter" do
|
||||||
|
now = 1_000_000
|
||||||
|
|
||||||
|
counters =
|
||||||
|
for _run <- 1..2 do
|
||||||
|
tracker = startTracker(logFactor: 10, seed: {3, 5, 7})
|
||||||
|
touchTimes(tracker, "hot", 250, now)
|
||||||
|
Tracker.counter(tracker, "hot", now)
|
||||||
|
end
|
||||||
|
|
||||||
|
assert [first, second] = counters
|
||||||
|
assert first == second
|
||||||
|
|
||||||
|
other = startTracker(logFactor: 10, seed: {11, 13, 17})
|
||||||
|
touchTimes(other, "hot", 250, now)
|
||||||
|
assert Tracker.counter(other, "hot", now) > 5
|
||||||
|
end
|
||||||
|
|
||||||
|
defp startTracker(opts) do
|
||||||
|
id = {Tracker, System.unique_integer([:positive])}
|
||||||
|
start_supervised!(Supervisor.child_spec({Tracker, opts}, id: id))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp touchTimes(tracker, key, count, now) do
|
||||||
|
Enum.each(1..count, fn _index -> Tracker.touch(tracker, key, now) end)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,626 @@
|
|||||||
|
defmodule Excache.IntrospectionTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Blocking
|
||||||
|
alias Excache.Command
|
||||||
|
alias Excache.Command.Introspection
|
||||||
|
alias Excache.Command.Introspection.Catalog
|
||||||
|
alias Excache.Command.Transaction
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
alias Excache.Memory
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Config.reset()
|
||||||
|
Runner.resetStats()
|
||||||
|
on_exit(&Config.reset/0)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "INFO" do
|
||||||
|
test "answers with every default section in the documented order", %{conn: conn} do
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO"])
|
||||||
|
|
||||||
|
assert sections(text) == [
|
||||||
|
"Server",
|
||||||
|
"Clients",
|
||||||
|
"Memory",
|
||||||
|
"Persistence",
|
||||||
|
"Stats",
|
||||||
|
"Replication",
|
||||||
|
"Keyspace"
|
||||||
|
]
|
||||||
|
|
||||||
|
assert String.ends_with?(text, "\r\n\r\n")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the server section reports the running system, not placeholders", %{conn: conn} do
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "server"])
|
||||||
|
assert sections(text) == ["Server"]
|
||||||
|
|
||||||
|
values = fields(text)
|
||||||
|
|
||||||
|
assert values["redis_mode"] == "standalone"
|
||||||
|
assert values["excache_version"] == to_string(Application.spec(:excache, :vsn))
|
||||||
|
assert values["run_id"] =~ ~r/^[0-9a-f]{40}$/
|
||||||
|
assert values["arch_bits"] == Integer.to_string(:erlang.system_info(:wordsize) * 8)
|
||||||
|
assert values["process_id"] == to_string(:os.getpid())
|
||||||
|
assert values["elixir_version"] == System.version()
|
||||||
|
assert values["otp_release"] == to_string(:erlang.system_info(:otp_release))
|
||||||
|
assert String.to_integer(values["uptime_in_seconds"]) >= 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "used_memory matches what the memory accountant reports", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["SET", "solo", String.duplicate("payload", 40)])
|
||||||
|
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "memory"])
|
||||||
|
values = fields(text)
|
||||||
|
|
||||||
|
assert sections(text) == ["Memory"]
|
||||||
|
assert String.to_integer(values["used_memory"]) == Memory.usedMemory()
|
||||||
|
assert values["maxmemory"] == "0"
|
||||||
|
assert values["maxmemory_policy"] == "noeviction"
|
||||||
|
assert values["used_memory_human"] =~ ~r/^[0-9.]+[BKMG]$/
|
||||||
|
|
||||||
|
dataset = String.to_integer(values["used_memory_dataset"])
|
||||||
|
overhead = String.to_integer(values["used_memory_overhead"])
|
||||||
|
assert dataset + overhead == String.to_integer(values["used_memory"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the keyspace section counts live keys, volatile keys and the mean ttl", %{conn: conn} do
|
||||||
|
assert {text, conn} = Client.command(conn, ["INFO", "keyspace"])
|
||||||
|
refute String.contains?(text, "db0")
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", "one"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "other", "two"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "timed", "three"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIRE", "timed", "120"])
|
||||||
|
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "keyspace"])
|
||||||
|
assert %{"db0" => summary} = fields(text)
|
||||||
|
|
||||||
|
assert [keys, expires, avgTtl] =
|
||||||
|
Regex.run(~r/keys=(\d+),expires=(\d+),avg_ttl=(\d+)/, summary,
|
||||||
|
capture: :all_but_first
|
||||||
|
)
|
||||||
|
|
||||||
|
assert keys == "3"
|
||||||
|
assert expires == "1"
|
||||||
|
assert String.to_integer(avgTtl) > 118_000
|
||||||
|
assert String.to_integer(avgTtl) <= 120_000
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the persistence section reads the real append only file", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "durable", "value"])
|
||||||
|
:ok = Aof.flush()
|
||||||
|
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "persistence"])
|
||||||
|
values = fields(text)
|
||||||
|
|
||||||
|
assert values["aof_enabled"] == "1"
|
||||||
|
assert values["aof_fsync_policy"] == "everysec"
|
||||||
|
assert values["loading"] == "0"
|
||||||
|
|
||||||
|
size = String.to_integer(values["aof_current_size"])
|
||||||
|
assert size > 0
|
||||||
|
assert size == File.stat!(Aof.path()).size
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the stats section counts connections and eviction refusals", %{conn: conn, port: port} do
|
||||||
|
assert {text, conn} = Client.command(conn, ["INFO", "stats"])
|
||||||
|
before = String.to_integer(fields(text)["total_connections_received"])
|
||||||
|
|
||||||
|
extra = Client.connect(port)
|
||||||
|
assert {{:simple, "PONG"}, _extra} = Client.command(extra, ["PING"])
|
||||||
|
|
||||||
|
assert {text, conn} = Client.command(conn, ["INFO", "stats"])
|
||||||
|
values = fields(text)
|
||||||
|
|
||||||
|
assert String.to_integer(values["total_connections_received"]) == before + 1
|
||||||
|
assert String.to_integer(values["total_net_input_bytes"]) > 0
|
||||||
|
assert values["evicted_keys"] == "0"
|
||||||
|
assert values["oom_rejections"] == "0"
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["SET", "filler", String.duplicate("x", 128)])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "1"])
|
||||||
|
assert {{:error, "OOM" <> _}, conn} = Client.command(conn, ["SET", "toobig", "value"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "0"])
|
||||||
|
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "stats"])
|
||||||
|
assert String.to_integer(fields(text)["oom_rejections"]) > 0
|
||||||
|
|
||||||
|
Client.close(extra)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the replication section tells the truth about being a lone master", %{conn: conn} do
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "replication"])
|
||||||
|
values = fields(text)
|
||||||
|
|
||||||
|
assert values["role"] == "master"
|
||||||
|
assert values["connected_slaves"] == "0"
|
||||||
|
assert values["master_failover_state"] == "no-failover"
|
||||||
|
assert values["repl_backlog_active"] == "0"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "INFO all and INFO everything add the cpu section", %{conn: conn} do
|
||||||
|
assert {plain, conn} = Client.command(conn, ["INFO"])
|
||||||
|
refute "Cpu" in sections(plain)
|
||||||
|
|
||||||
|
assert {all, conn} = Client.command(conn, ["INFO", "all"])
|
||||||
|
assert "Cpu" in sections(all)
|
||||||
|
|
||||||
|
assert {everything, conn} = Client.command(conn, ["INFO", "everything"])
|
||||||
|
assert sections(everything) == sections(all)
|
||||||
|
|
||||||
|
values = fields(all)
|
||||||
|
assert String.to_integer(values["schedulers"]) == :erlang.system_info(:schedulers)
|
||||||
|
assert values["used_cpu_total"] =~ ~r/^\d+\.\d\d$/
|
||||||
|
|
||||||
|
assert {defaults, _conn} = Client.command(conn, ["INFO", "default"])
|
||||||
|
assert sections(defaults) == sections(plain)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "several sections can be asked for at once and unknown ones are dropped", %{conn: conn} do
|
||||||
|
assert {text, conn} = Client.command(conn, ["INFO", "replication", "clients"])
|
||||||
|
assert sections(text) == ["Replication", "Clients"]
|
||||||
|
|
||||||
|
assert {"", conn} = Client.command(conn, ["INFO", "nosuchsection"])
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "clients", "nosuchsection"])
|
||||||
|
assert sections(text) == ["Clients"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "blocked_clients counts a connection parked in BLPOP", %{conn: conn, port: port} do
|
||||||
|
assert clientField(conn, "blocked_clients") == 0
|
||||||
|
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "parked", "0"])
|
||||||
|
awaitUntil(fn -> Blocking.waitingCount("parked") == 1 end)
|
||||||
|
|
||||||
|
assert clientField(conn, "blocked_clients") == 1
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "parked", "wake"])
|
||||||
|
assert Task.await(waiter) == ["parked", "wake"]
|
||||||
|
awaitUntil(fn -> clientField(conn, "blocked_clients") == 0 end)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "pubsub_clients and pubsub_channels follow real subscriptions", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
assert clientField(conn, "pubsub_clients") == 0
|
||||||
|
|
||||||
|
listener = Client.connect(port)
|
||||||
|
|
||||||
|
assert {["subscribe", "news", 1], _listener} =
|
||||||
|
Client.command(listener, ["SUBSCRIBE", "news"])
|
||||||
|
|
||||||
|
awaitUntil(fn -> clientField(conn, "pubsub_clients") == 1 end)
|
||||||
|
|
||||||
|
assert {text, _conn} = Client.command(conn, ["INFO", "stats"])
|
||||||
|
assert fields(text)["pubsub_channels"] == "1"
|
||||||
|
assert fields(text)["pubsub_patterns"] == "0"
|
||||||
|
|
||||||
|
Client.close(listener)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "CLIENT" do
|
||||||
|
test "ID hands out a distinct rising number per connection", %{conn: conn, port: port} do
|
||||||
|
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
assert is_integer(mine) and mine > 0
|
||||||
|
|
||||||
|
other = Client.connect(port)
|
||||||
|
assert {theirs, _other} = Client.command(other, ["CLIENT", "ID"])
|
||||||
|
assert theirs > mine
|
||||||
|
|
||||||
|
assert {^mine, _conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
Client.close(other)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SETNAME and GETNAME round trip and reject illegal names", %{conn: conn} do
|
||||||
|
assert {"", conn} = Client.command(conn, ["CLIENT", "GETNAME"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "SETNAME", "importer"])
|
||||||
|
assert {"importer", conn} = Client.command(conn, ["CLIENT", "GETNAME"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Client names cannot contain spaces" <> _}, conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "SETNAME", "two words"])
|
||||||
|
|
||||||
|
assert {"importer", conn} = Client.command(conn, ["CLIENT", "GETNAME"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "SETNAME", ""])
|
||||||
|
assert {"", _conn} = Client.command(conn, ["CLIENT", "GETNAME"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LIST shows one line per connection with the real address and name", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
|
||||||
|
other = Client.connect(port)
|
||||||
|
assert {theirs, other} = Client.command(other, ["CLIENT", "ID"])
|
||||||
|
assert {{:simple, "OK"}, _other} = Client.command(other, ["CLIENT", "SETNAME", "worker"])
|
||||||
|
|
||||||
|
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST"])
|
||||||
|
lines = listLines(text)
|
||||||
|
|
||||||
|
assert %{"name" => "worker", "addr" => address} = lines[theirs]
|
||||||
|
assert address =~ ~r/^127\.0\.0\.1:\d+$/
|
||||||
|
assert lines[theirs]["laddr"] =~ ~r/^127\.0\.0\.1:#{port}$/
|
||||||
|
assert lines[mine]["name"] == ""
|
||||||
|
assert lines[mine]["db"] == "0"
|
||||||
|
assert String.to_integer(lines[mine]["tot-net-in"]) > 0
|
||||||
|
|
||||||
|
assert {filtered, conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "LIST", "ID", Integer.to_string(theirs)])
|
||||||
|
|
||||||
|
assert Map.keys(listLines(filtered)) == [theirs]
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "LIST", "NOPE", "1"])
|
||||||
|
|
||||||
|
Client.close(other)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LIST TYPE separates subscribers from plain connections", %{conn: conn, port: port} do
|
||||||
|
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
|
||||||
|
listener = Client.connect(port)
|
||||||
|
assert {theirs, listener} = Client.command(listener, ["CLIENT", "ID"])
|
||||||
|
|
||||||
|
assert {["subscribe", "feed", 1], _listener} =
|
||||||
|
Client.command(listener, ["SUBSCRIBE", "feed"])
|
||||||
|
|
||||||
|
awaitUntil(fn ->
|
||||||
|
{text, _conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "pubsub"])
|
||||||
|
Map.has_key?(listLines(text), theirs)
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "pubsub"])
|
||||||
|
assert Map.keys(listLines(text)) == [theirs]
|
||||||
|
assert listLines(text)[theirs]["sub"] == "1"
|
||||||
|
|
||||||
|
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "normal"])
|
||||||
|
assert Map.has_key?(listLines(text), mine)
|
||||||
|
refute Map.has_key?(listLines(text), theirs)
|
||||||
|
|
||||||
|
assert {"", conn} = Client.command(conn, ["CLIENT", "LIST", "TYPE", "replica"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "LIST", "TYPE", "nonsense"])
|
||||||
|
|
||||||
|
Client.close(listener)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "INFO describes the calling connection only", %{conn: conn} do
|
||||||
|
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "SETNAME", "solo"])
|
||||||
|
assert {text, _conn} = Client.command(conn, ["CLIENT", "INFO"])
|
||||||
|
|
||||||
|
assert [{id, fieldMap}] = Map.to_list(listLines(text <> "\n"))
|
||||||
|
assert id == mine
|
||||||
|
assert fieldMap["name"] == "solo"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "idle grows while a connection stays quiet and age tracks the connect time", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
quiet = Client.connect(port)
|
||||||
|
assert {quietId, _quiet} = Client.command(quiet, ["CLIENT", "ID"])
|
||||||
|
|
||||||
|
Process.sleep(1_200)
|
||||||
|
|
||||||
|
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
assert {text, _conn} = Client.command(conn, ["CLIENT", "LIST"])
|
||||||
|
lines = listLines(text)
|
||||||
|
|
||||||
|
assert String.to_integer(lines[quietId]["idle"]) >= 1
|
||||||
|
assert String.to_integer(lines[quietId]["age"]) >= 1
|
||||||
|
assert lines[mine]["idle"] == "0"
|
||||||
|
|
||||||
|
Client.close(quiet)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "KILL by ID tears down another live connection", %{conn: conn, port: port} do
|
||||||
|
victim = Client.connect(port)
|
||||||
|
assert {victimId, victim} = Client.command(victim, ["CLIENT", "ID"])
|
||||||
|
assert {{:simple, "PONG"}, victim} = Client.command(victim, ["PING"])
|
||||||
|
|
||||||
|
assert {1, conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "KILL", "ID", Integer.to_string(victimId)])
|
||||||
|
|
||||||
|
assert {:error, :closed} = :gen_tcp.recv(victim.socket, 0, 1_000)
|
||||||
|
|
||||||
|
assert {0, conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "KILL", "ID", Integer.to_string(victimId)])
|
||||||
|
|
||||||
|
assert {text, _conn} = Client.command(conn, ["CLIENT", "LIST"])
|
||||||
|
refute Map.has_key?(listLines(text), victimId)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "KILL by address accepts the legacy single argument form", %{conn: conn, port: port} do
|
||||||
|
victim = Client.connect(port)
|
||||||
|
assert {victimId, victim} = Client.command(victim, ["CLIENT", "ID"])
|
||||||
|
|
||||||
|
assert {text, conn} = Client.command(conn, ["CLIENT", "LIST"])
|
||||||
|
address = listLines(text)[victimId]["addr"]
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CLIENT", "KILL", address])
|
||||||
|
assert {:error, :closed} = :gen_tcp.recv(victim.socket, 0, 1_000)
|
||||||
|
|
||||||
|
assert {{:error, "ERR No such client address in the client list"}, _conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "KILL", address])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "KILL skips the caller unless SKIPME no is given", %{conn: conn, port: port} do
|
||||||
|
assert {mine, conn} = Client.command(conn, ["CLIENT", "ID"])
|
||||||
|
bystander = Client.connect(port)
|
||||||
|
assert {_id, _bystander} = Client.command(bystander, ["CLIENT", "ID"])
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["CLIENT", "KILL", "ID", Integer.to_string(mine)])
|
||||||
|
assert {{:simple, "PONG"}, conn} = Client.command(conn, ["PING"])
|
||||||
|
|
||||||
|
assert {1, conn} =
|
||||||
|
Client.command(conn, [
|
||||||
|
"CLIENT",
|
||||||
|
"KILL",
|
||||||
|
"ID",
|
||||||
|
Integer.to_string(mine),
|
||||||
|
"SKIPME",
|
||||||
|
"no"
|
||||||
|
])
|
||||||
|
|
||||||
|
assert {:error, :closed} = :gen_tcp.recv(conn.socket, 0, 1_000)
|
||||||
|
Client.close(bystander)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "KILL rejects filters it does not understand", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "KILL", "WHATEVER", "1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "KILL", "ID", "notanumber"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Unknown subcommand" <> _}, _conn} =
|
||||||
|
Client.command(conn, ["CLIENT", "NOSUCHTHING"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "HELP lists the supported subcommands", %{conn: conn} do
|
||||||
|
assert {lines, _conn} = Client.command(conn, ["CLIENT", "HELP"])
|
||||||
|
assert {:simple, "KILL <ip:port>"} in lines
|
||||||
|
assert {:simple, "ID"} in lines
|
||||||
|
end
|
||||||
|
|
||||||
|
test "CLIENT off a real connection reports that it needs one" do
|
||||||
|
assert {[{:error, "ERR CLIENT is only available on a server connection"}], _state} =
|
||||||
|
Introspection.dispatch("CLIENT", ["ID"], commandState())
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "COMMAND" do
|
||||||
|
test "the catalog covers exactly the commands the server dispatches" do
|
||||||
|
known = MapSet.new(Command.known_commands())
|
||||||
|
assert MapSet.new(Catalog.tableNames()) == known
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the full reply carries one six element entry per command" do
|
||||||
|
assert {[{:array, entries}], _state} = Introspection.dispatch("COMMAND", [], commandState())
|
||||||
|
|
||||||
|
assert length(entries) == length(Enum.uniq(Command.known_commands()))
|
||||||
|
|
||||||
|
Enum.each(entries, fn {:array, entry} ->
|
||||||
|
assert [{:bulk, name}, {:integer, arity}, {:array, flags}, first, last, step] = entry
|
||||||
|
assert name == String.downcase(name)
|
||||||
|
assert arity != 0
|
||||||
|
assert Enum.all?(flags, &match?({:simple, _flag}, &1))
|
||||||
|
assert match?({:integer, _value}, first)
|
||||||
|
assert match?({:integer, _value}, last)
|
||||||
|
assert match?({:integer, _value}, step)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "COUNT and LIST agree with the full reply" do
|
||||||
|
assert {[{:integer, count}], _state} =
|
||||||
|
Introspection.dispatch("COMMAND", ["COUNT"], commandState())
|
||||||
|
|
||||||
|
assert {[{:array, names}], _state} =
|
||||||
|
Introspection.dispatch("COMMAND", ["LIST"], commandState())
|
||||||
|
|
||||||
|
assert count == length(names)
|
||||||
|
assert count == length(Enum.uniq(Command.known_commands()))
|
||||||
|
assert {:bulk, "append"} in names
|
||||||
|
end
|
||||||
|
|
||||||
|
test "INFO gives the real arity and key positions per command" do
|
||||||
|
assert commandInfo("GET") == {"get", 2, ["readonly", "fast"], 1, 1, 1}
|
||||||
|
assert commandInfo("SET") == {"set", -3, ["write", "denyoom"], 1, 1, 1}
|
||||||
|
assert commandInfo("MSET") == {"mset", -3, ["write", "denyoom"], 1, -1, 2}
|
||||||
|
assert commandInfo("DEL") == {"del", -2, ["write"], 1, -1, 1}
|
||||||
|
assert commandInfo("BLPOP") == {"blpop", -3, ["write", "noscript", "blocking"], 1, -2, 1}
|
||||||
|
assert commandInfo("BITOP") == {"bitop", -4, ["write", "denyoom"], 2, -1, 1}
|
||||||
|
assert commandInfo("SMOVE") == {"smove", 4, ["write", "fast"], 1, 2, 1}
|
||||||
|
assert commandInfo("PING") == {"ping", -1, ["fast"], 0, 0, 0}
|
||||||
|
|
||||||
|
assert {"xread", -4, flags, 0, 0, 0} = commandInfo("XREAD")
|
||||||
|
assert "movablekeys" in flags
|
||||||
|
end
|
||||||
|
|
||||||
|
test "INFO is case insensitive and answers nil for what it does not have" do
|
||||||
|
assert {[{:array, [entry, nil?]}], _state} =
|
||||||
|
Introspection.dispatch("COMMAND", ["INFO", "get", "nosuchcommand"], commandState())
|
||||||
|
|
||||||
|
assert {:array, [{:bulk, "get"} | _rest]} = entry
|
||||||
|
assert nil? == {:array, nil}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the declared arity matches what the dispatcher actually accepts", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'get' command"}, conn} =
|
||||||
|
Client.command(conn, ["GET", "a", "b"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'smove' command"}, conn} =
|
||||||
|
Client.command(conn, ["SMOVE", "a", "b"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'linsert' command"}, _conn} =
|
||||||
|
Client.command(conn, ["LINSERT", "a", "BEFORE", "b"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "GETKEYS walks plain ranges" do
|
||||||
|
assert getkeys(["GET", "one"]) == ["one"]
|
||||||
|
assert getkeys(["MSET", "a", "1", "b", "2", "c", "3"]) == ["a", "b", "c"]
|
||||||
|
assert getkeys(["DEL", "a", "b", "c"]) == ["a", "b", "c"]
|
||||||
|
assert getkeys(["BLPOP", "left", "right", "0"]) == ["left", "right"]
|
||||||
|
assert getkeys(["BITOP", "AND", "dest", "src1", "src2"]) == ["dest", "src1", "src2"]
|
||||||
|
assert getkeys(["SMOVE", "from", "to", "member"]) == ["from", "to"]
|
||||||
|
assert getkeys(["EXPIRE", "solo", "10"]) == ["solo"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "GETKEYS resolves the commands whose keys move" do
|
||||||
|
assert getkeys(["XREAD", "COUNT", "2", "STREAMS", "s1", "s2", "0", "0"]) == ["s1", "s2"]
|
||||||
|
|
||||||
|
assert getkeys([
|
||||||
|
"XREADGROUP",
|
||||||
|
"GROUP",
|
||||||
|
"g",
|
||||||
|
"c",
|
||||||
|
"STREAMS",
|
||||||
|
"stream",
|
||||||
|
">"
|
||||||
|
]) == ["stream"]
|
||||||
|
|
||||||
|
assert getkeys(["SINTERCARD", "2", "a", "b", "LIMIT", "1"]) == ["a", "b"]
|
||||||
|
assert getkeys(["BLMPOP", "0", "2", "a", "b", "LEFT"]) == ["a", "b"]
|
||||||
|
assert getkeys(["OBJECT", "ENCODING", "thing"]) == ["thing"]
|
||||||
|
assert getkeys(["MEMORY", "USAGE", "thing"]) == ["thing"]
|
||||||
|
assert getkeys(["XINFO", "STREAM", "events"]) == ["events"]
|
||||||
|
assert getkeys(["XGROUP", "CREATE", "events", "group", "$"]) == ["events"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "GETKEYS refuses what it cannot answer" do
|
||||||
|
assert getkeys(["PING"]) == {:error, "ERR The command has no key arguments"}
|
||||||
|
assert getkeys(["MEMORY", "DOCTOR"]) == {:error, "ERR The command has no key arguments"}
|
||||||
|
assert getkeys(["NOSUCHCOMMAND", "a"]) == {:error, "ERR Invalid command specified"}
|
||||||
|
|
||||||
|
assert getkeys(["GET"]) ==
|
||||||
|
{:error, "ERR Invalid number of arguments specified for command"}
|
||||||
|
|
||||||
|
assert getkeys(["XREAD", "COUNT", "2", "s1", "0"]) ==
|
||||||
|
{:error, "ERR Invalid arguments specified for command"}
|
||||||
|
|
||||||
|
assert getkeys(["SINTERCARD", "3", "a", "b"]) ==
|
||||||
|
{:error, "ERR Invalid arguments specified for command"}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "DOCS answers with the group and arity taken from the same table" do
|
||||||
|
assert {[{:array, entries}], _state} =
|
||||||
|
Introspection.dispatch(
|
||||||
|
"COMMAND",
|
||||||
|
["DOCS", "zadd", "nosuchcommand"],
|
||||||
|
commandState()
|
||||||
|
)
|
||||||
|
|
||||||
|
assert entries == [
|
||||||
|
{:bulk, "zadd"},
|
||||||
|
{:array, [{:bulk, "group"}, {:bulk, "zset"}, {:bulk, "arity"}, {:integer, -4}]}
|
||||||
|
]
|
||||||
|
|
||||||
|
assert {[{:array, everything}], _state} =
|
||||||
|
Introspection.dispatch("COMMAND", ["DOCS"], commandState())
|
||||||
|
|
||||||
|
assert length(everything) == 2 * length(Enum.uniq(Command.known_commands()))
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an unknown subcommand is named in the error" do
|
||||||
|
assert {[{:error, message}], _state} =
|
||||||
|
Introspection.dispatch("COMMAND", ["NOPE"], commandState())
|
||||||
|
|
||||||
|
assert message =~ "Unknown subcommand or wrong number of arguments for 'NOPE'"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the extended dispatcher already routes COMMAND to this module" do
|
||||||
|
assert {[{:integer, count}], _state} =
|
||||||
|
Excache.Command.Extended.dispatch("COMMAND", ["COUNT"], commandState())
|
||||||
|
|
||||||
|
assert count == Catalog.count()
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp commandState,
|
||||||
|
do: Map.put(Transaction.initial_state(), :subscriptions, MapSet.new())
|
||||||
|
|
||||||
|
defp commandInfo(name) do
|
||||||
|
assert {[{:array, [{:array, entry}]}], _state} =
|
||||||
|
Introspection.dispatch("COMMAND", ["INFO", name], commandState())
|
||||||
|
|
||||||
|
[
|
||||||
|
{:bulk, lowered},
|
||||||
|
{:integer, arity},
|
||||||
|
{:array, flags},
|
||||||
|
{:integer, first},
|
||||||
|
{:integer, last},
|
||||||
|
{:integer, step}
|
||||||
|
] = entry
|
||||||
|
|
||||||
|
{lowered, arity, Enum.map(flags, fn {:simple, flag} -> flag end), first, last, step}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp getkeys(argv) do
|
||||||
|
case Introspection.dispatch("COMMAND", ["GETKEYS" | argv], commandState()) do
|
||||||
|
{[{:array, keys}], _state} -> Enum.map(keys, fn {:bulk, key} -> key end)
|
||||||
|
{[error], _state} -> error
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp clientField(conn, name) do
|
||||||
|
{text, _conn} = Client.command(conn, ["INFO", "clients"])
|
||||||
|
String.to_integer(fields(text)[name])
|
||||||
|
end
|
||||||
|
|
||||||
|
defp sections(text) do
|
||||||
|
for "# " <> title <- String.split(text, "\r\n"), do: title
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fields(text) do
|
||||||
|
text
|
||||||
|
|> String.split("\r\n")
|
||||||
|
|> Enum.reject(fn line -> line == "" or String.starts_with?(line, "#") end)
|
||||||
|
|> Map.new(fn line ->
|
||||||
|
[key, value] = String.split(line, ":", parts: 2)
|
||||||
|
{key, value}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp listLines(text) do
|
||||||
|
text
|
||||||
|
|> String.split("\n")
|
||||||
|
|> Enum.reject(&(&1 == ""))
|
||||||
|
|> Map.new(fn line ->
|
||||||
|
parsed =
|
||||||
|
Map.new(String.split(line, " "), fn pair ->
|
||||||
|
[key, value] = String.split(pair, "=", parts: 2)
|
||||||
|
{key, value}
|
||||||
|
end)
|
||||||
|
|
||||||
|
{String.to_integer(parsed["id"]), parsed}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp blockingTask(port, args) do
|
||||||
|
Task.async(fn ->
|
||||||
|
client = Client.connect(port)
|
||||||
|
{reply, _client} = Client.command(client, args)
|
||||||
|
Client.close(client)
|
||||||
|
reply
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp awaitUntil(check, attempts \\ 200) do
|
||||||
|
cond do
|
||||||
|
check.() -> :ok
|
||||||
|
attempts == 0 -> flunk("condition never held")
|
||||||
|
true -> Process.sleep(10) && awaitUntil(check, attempts - 1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
defmodule Excache.KeyspaceScanTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
test "SCAN pages through the whole keyspace and returns every key once", %{conn: conn} do
|
||||||
|
conn = seed(conn, "key", 1..120)
|
||||||
|
|
||||||
|
{keys, calls, conn} = drain(conn, fn cursor -> ["SCAN", cursor] end)
|
||||||
|
|
||||||
|
assert calls > 1
|
||||||
|
assert length(keys) == 120
|
||||||
|
assert Enum.sort(keys) == Enum.sort(Enum.map(1..120, &"key:#{&1}"))
|
||||||
|
|
||||||
|
{[cursor, batch], _conn} = Client.command(conn, ["SCAN", "0", "COUNT", "1000"])
|
||||||
|
assert cursor == "0"
|
||||||
|
assert length(batch) == 120
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SCAN filters with MATCH and TYPE", %{conn: conn} do
|
||||||
|
conn = seed(conn, "user", 1..30)
|
||||||
|
conn = seed(conn, "session", 1..30)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "user:list", "a"])
|
||||||
|
|
||||||
|
{users, _calls, conn} = drain(conn, fn cursor -> ["SCAN", cursor, "MATCH", "user:*"] end)
|
||||||
|
assert length(users) == 31
|
||||||
|
|
||||||
|
{strings, _calls, conn} =
|
||||||
|
drain(conn, fn cursor -> ["SCAN", cursor, "MATCH", "user:*", "TYPE", "string"] end)
|
||||||
|
|
||||||
|
assert length(strings) == 30
|
||||||
|
refute "user:list" in strings
|
||||||
|
|
||||||
|
{lists, _calls, conn} = drain(conn, fn cursor -> ["SCAN", cursor, "TYPE", "list"] end)
|
||||||
|
assert lists == ["user:list"]
|
||||||
|
|
||||||
|
{none, _calls, _conn} = drain(conn, fn cursor -> ["SCAN", cursor, "MATCH", "nothing:*"] end)
|
||||||
|
assert none == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SCAN returns keys that live through the whole iteration", %{conn: conn} do
|
||||||
|
conn = seed(conn, "stable", 1..80)
|
||||||
|
|
||||||
|
{[cursor, first], conn} = Client.command(conn, ["SCAN", "0"])
|
||||||
|
refute cursor == "0"
|
||||||
|
|
||||||
|
conn = seed(conn, "noise", 1..60)
|
||||||
|
assert {1, conn} = Client.command(conn, ["DEL", "noise:1"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["DEL", "noise:2"])
|
||||||
|
|
||||||
|
{seen, _calls, _conn} = drainFrom(conn, fn value -> ["SCAN", value] end, cursor, first)
|
||||||
|
|
||||||
|
stable = Enum.filter(seen, &String.starts_with?(&1, "stable:"))
|
||||||
|
assert length(stable) == 80
|
||||||
|
assert length(Enum.uniq(stable)) == 80
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SCAN validates the cursor", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR invalid cursor"}, conn} = Client.command(conn, ["SCAN", "abc"])
|
||||||
|
assert {{:error, "ERR invalid cursor"}, conn} = Client.command(conn, ["SCAN", "-1"])
|
||||||
|
|
||||||
|
{[cursor, keys], conn} = Client.command(conn, ["SCAN", "9999"])
|
||||||
|
assert cursor == "0"
|
||||||
|
assert keys == []
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["SCAN", "0", "COUNT", "0"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
|
||||||
|
Client.command(conn, ["SCAN", "0", "COUNT", "many"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} = Client.command(conn, ["SCAN", "0", "NOVALUES"])
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} = Client.command(conn, ["SCAN", "0", "MATCH"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'scan' command"}, _conn} =
|
||||||
|
Client.command(conn, ["SCAN"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "HSCAN returns field value pairs, MATCH and NOVALUES", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["HSET", "h", "f1", "v1", "f2", "v2", "other", "v3"])
|
||||||
|
|
||||||
|
{[cursor, flat], conn} = Client.command(conn, ["HSCAN", "h", "0"])
|
||||||
|
assert cursor == "0"
|
||||||
|
|
||||||
|
assert flat |> Enum.chunk_every(2) |> Enum.sort() == [
|
||||||
|
["f1", "v1"],
|
||||||
|
["f2", "v2"],
|
||||||
|
["other", "v3"]
|
||||||
|
]
|
||||||
|
|
||||||
|
{[_cursor, fields], conn} = Client.command(conn, ["HSCAN", "h", "0", "NOVALUES"])
|
||||||
|
assert Enum.sort(fields) == ["f1", "f2", "other"]
|
||||||
|
|
||||||
|
{[_cursor, matched], conn} = Client.command(conn, ["HSCAN", "h", "0", "MATCH", "f*"])
|
||||||
|
assert matched |> Enum.chunk_every(2) |> Enum.sort() == [["f1", "v1"], ["f2", "v2"]]
|
||||||
|
|
||||||
|
{[_cursor, names], conn} =
|
||||||
|
Client.command(conn, ["HSCAN", "h", "0", "MATCH", "f?", "NOVALUES"])
|
||||||
|
|
||||||
|
assert Enum.sort(names) == ["f1", "f2"]
|
||||||
|
|
||||||
|
{[emptyCursor, empty], conn} = Client.command(conn, ["HSCAN", "nohash", "0"])
|
||||||
|
assert emptyCursor == "0"
|
||||||
|
assert empty == []
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "text", "v"])
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["HSCAN", "text", "0"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["HSCAN", "h", "0", "TYPE", "hash"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR invalid cursor"}, _conn} = Client.command(conn, ["HSCAN", "h", "later"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "HSCAN pages through a large hash exactly once per field", %{conn: conn} do
|
||||||
|
fields = Enum.flat_map(1..300, fn index -> ["field:#{index}", "value:#{index}"] end)
|
||||||
|
assert {300, conn} = Client.command(conn, ["HSET", "big"] ++ fields)
|
||||||
|
|
||||||
|
{flat, calls, conn} = drain(conn, fn cursor -> ["HSCAN", "big", cursor] end)
|
||||||
|
pairs = Enum.chunk_every(flat, 2)
|
||||||
|
|
||||||
|
assert calls > 1
|
||||||
|
assert length(pairs) == 300
|
||||||
|
assert length(Enum.uniq(pairs)) == 300
|
||||||
|
assert ["field:7", "value:7"] in pairs
|
||||||
|
|
||||||
|
{sparse, sparseCalls, _conn} =
|
||||||
|
drain(conn, fn cursor -> ["HSCAN", "big", cursor, "COUNT", "500"] end)
|
||||||
|
|
||||||
|
assert sparseCalls == 1
|
||||||
|
assert length(sparse) == 600
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SSCAN pages a set and keeps members that live through the iteration", %{conn: conn} do
|
||||||
|
members = Enum.map(1..200, &"member:#{&1}")
|
||||||
|
assert {200, conn} = Client.command(conn, ["SADD", "bag"] ++ members)
|
||||||
|
|
||||||
|
{[cursor, first], conn} = Client.command(conn, ["SSCAN", "bag", "0"])
|
||||||
|
refute cursor == "0"
|
||||||
|
|
||||||
|
assert {50, conn} = Client.command(conn, ["SADD", "bag"] ++ Enum.map(1..50, &"extra:#{&1}"))
|
||||||
|
assert {25, conn} = Client.command(conn, ["SREM", "bag"] ++ Enum.map(1..25, &"extra:#{&1}"))
|
||||||
|
|
||||||
|
{seen, _calls, conn} = drainFrom(conn, fn value -> ["SSCAN", "bag", value] end, cursor, first)
|
||||||
|
|
||||||
|
stable = Enum.filter(seen, &String.starts_with?(&1, "member:"))
|
||||||
|
assert length(stable) == 200
|
||||||
|
assert length(Enum.uniq(stable)) == 200
|
||||||
|
|
||||||
|
{[_cursor, matched], conn} =
|
||||||
|
Client.command(conn, ["SSCAN", "bag", "0", "MATCH", "member:1", "COUNT", "1000"])
|
||||||
|
|
||||||
|
assert matched == ["member:1"]
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["SSCAN", "bag", "0", "NOVALUES"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "ZSCAN returns members with their scores", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["ZADD", "board", "1", "a", "2.5", "b", "-3", "c"])
|
||||||
|
|
||||||
|
{[cursor, flat], conn} = Client.command(conn, ["ZSCAN", "board", "0"])
|
||||||
|
assert cursor == "0"
|
||||||
|
|
||||||
|
assert flat |> Enum.chunk_every(2) |> Enum.sort() == [["a", "1"], ["b", "2.5"], ["c", "-3"]]
|
||||||
|
|
||||||
|
{[_cursor, matched], conn} = Client.command(conn, ["ZSCAN", "board", "0", "MATCH", "a"])
|
||||||
|
assert matched == ["a", "1"]
|
||||||
|
|
||||||
|
{[emptyCursor, empty], conn} = Client.command(conn, ["ZSCAN", "nozset", "0"])
|
||||||
|
assert emptyCursor == "0"
|
||||||
|
assert empty == []
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "listkey", "x"])
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, _conn} = Client.command(conn, ["ZSCAN", "listkey", "0"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "ZSCAN pages a large sorted set exactly once per member", %{conn: conn} do
|
||||||
|
entries = Enum.flat_map(1..250, fn index -> [Integer.to_string(index), "member:#{index}"] end)
|
||||||
|
assert {250, conn} = Client.command(conn, ["ZADD", "ranking"] ++ entries)
|
||||||
|
|
||||||
|
{flat, calls, _conn} = drain(conn, fn cursor -> ["ZSCAN", "ranking", cursor] end)
|
||||||
|
pairs = Enum.chunk_every(flat, 2)
|
||||||
|
|
||||||
|
assert calls > 1
|
||||||
|
assert length(pairs) == 250
|
||||||
|
assert length(Enum.uniq(pairs)) == 250
|
||||||
|
assert ["member:42", "42"] in pairs
|
||||||
|
end
|
||||||
|
|
||||||
|
defp seed(conn, prefix, range) do
|
||||||
|
Enum.reduce(range, conn, fn index, current ->
|
||||||
|
{{:simple, "OK"}, next} = Client.command(current, ["SET", "#{prefix}:#{index}", "v"])
|
||||||
|
next
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp drain(conn, buildArgs), do: drainFrom(conn, buildArgs, "0", [])
|
||||||
|
|
||||||
|
defp drainFrom(conn, buildArgs, cursor, acc), do: drainFrom(conn, buildArgs, cursor, acc, 0)
|
||||||
|
|
||||||
|
defp drainFrom(_conn, _buildArgs, _cursor, _acc, calls) when calls > 500,
|
||||||
|
do: flunk("scan never reached cursor 0")
|
||||||
|
|
||||||
|
defp drainFrom(conn, buildArgs, cursor, acc, calls) do
|
||||||
|
{[next, batch], conn} = Client.command(conn, buildArgs.(cursor))
|
||||||
|
collected = acc ++ batch
|
||||||
|
|
||||||
|
if next == "0" do
|
||||||
|
{collected, calls + 1, conn}
|
||||||
|
else
|
||||||
|
drainFrom(conn, buildArgs, next, collected, calls + 1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,548 @@
|
|||||||
|
defmodule Excache.KeyspaceTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
test "STRLEN, GETRANGE and SETRANGE follow Redis offsets", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "greeting", "Hello World"])
|
||||||
|
assert {11, conn} = Client.command(conn, ["STRLEN", "greeting"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["STRLEN", "absent"])
|
||||||
|
|
||||||
|
assert {"Hello", conn} = Client.command(conn, ["GETRANGE", "greeting", "0", "4"])
|
||||||
|
assert {"World", conn} = Client.command(conn, ["GETRANGE", "greeting", "-5", "-1"])
|
||||||
|
assert {"", conn} = Client.command(conn, ["GETRANGE", "greeting", "10", "4"])
|
||||||
|
assert {"", conn} = Client.command(conn, ["GETRANGE", "absent", "0", "-1"])
|
||||||
|
|
||||||
|
assert {11, conn} = Client.command(conn, ["SETRANGE", "greeting", "6", "Redis"])
|
||||||
|
assert {"Hello Redis", conn} = Client.command(conn, ["GETRANGE", "greeting", "0", "-1"])
|
||||||
|
|
||||||
|
assert {8, conn} = Client.command(conn, ["SETRANGE", "padded", "5", "abc"])
|
||||||
|
{padded, conn} = Client.command(conn, ["GET", "padded"])
|
||||||
|
assert padded == :binary.copy(<<0>>, 5) <> "abc"
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["SETRANGE", "untouched", "0", ""])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "untouched"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR offset is out of range"}, _conn} =
|
||||||
|
Client.command(conn, ["SETRANGE", "greeting", "-1", "x"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SETNX and GETDEL", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["SETNX", "slot", "first"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["SETNX", "slot", "second"])
|
||||||
|
assert {"first", conn} = Client.command(conn, ["GET", "slot"])
|
||||||
|
|
||||||
|
assert {"first", conn} = Client.command(conn, ["GETDEL", "slot"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "slot"])
|
||||||
|
assert {nil, conn} = Client.command(conn, ["GETDEL", "slot"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "queue", "x"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, _conn} = Client.command(conn, ["GETDEL", "queue"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "INCRBYFLOAT accumulates and rejects non numeric values", %{conn: conn} do
|
||||||
|
assert {"10.5", conn} = Client.command(conn, ["INCRBYFLOAT", "balance", "10.5"])
|
||||||
|
assert {"10.6", conn} = Client.command(conn, ["INCRBYFLOAT", "balance", "0.1"])
|
||||||
|
assert {"8", conn} = Client.command(conn, ["INCRBYFLOAT", "balance", "-2.6"])
|
||||||
|
assert {"5", conn} = Client.command(conn, ["INCRBYFLOAT", "counter", "5"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not a valid float"}, conn} =
|
||||||
|
Client.command(conn, ["INCRBYFLOAT", "balance", "abc"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "word", "hello"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not a valid float"}, _conn} =
|
||||||
|
Client.command(conn, ["INCRBYFLOAT", "word", "1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "HSETNX and HMGET", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["HSETNX", "profile", "name", "ann"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["HSETNX", "profile", "name", "ben"])
|
||||||
|
assert {"ann", conn} = Client.command(conn, ["HGET", "profile", "name"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["HSETNX", "profile", "city", "gdansk"])
|
||||||
|
|
||||||
|
{values, conn} = Client.command(conn, ["HMGET", "profile", "name", "ghost", "city"])
|
||||||
|
assert values == ["ann", nil, "gdansk"]
|
||||||
|
|
||||||
|
{absent, conn} = Client.command(conn, ["HMGET", "nohash", "a", "b"])
|
||||||
|
assert absent == [nil, nil]
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, _conn} = Client.command(conn, ["HMGET", "plain", "a"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SMISMEMBER and SINTERCARD", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["SADD", "left", "a", "b", "c"])
|
||||||
|
assert {3, conn} = Client.command(conn, ["SADD", "right", "b", "c", "d"])
|
||||||
|
|
||||||
|
{flags, conn} = Client.command(conn, ["SMISMEMBER", "left", "a", "zz", "c"])
|
||||||
|
assert flags == [1, 0, 1]
|
||||||
|
|
||||||
|
{none, conn} = Client.command(conn, ["SMISMEMBER", "noset", "a"])
|
||||||
|
assert none == [0]
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["SINTERCARD", "2", "left", "right"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["SINTERCARD", "2", "left", "right", "LIMIT", "0"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["SINTERCARD", "2", "left", "right", "LIMIT", "1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR numkeys should be greater than 0"}, conn} =
|
||||||
|
Client.command(conn, ["SINTERCARD", "0", "left"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Number of keys can't be greater than number of args"}, conn} =
|
||||||
|
Client.command(conn, ["SINTERCARD", "3", "left"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR LIMIT can't be negative"}, conn} =
|
||||||
|
Client.command(conn, ["SINTERCARD", "1", "left", "LIMIT", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["SINTERCARD", "1", "left", "BOUND", "1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SPOP replies with a bulk without count and an array with count", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["SADD", "bag", "a", "b", "c"])
|
||||||
|
|
||||||
|
{single, conn} = Client.command(conn, ["SPOP", "bag"])
|
||||||
|
assert single in ["a", "b", "c"]
|
||||||
|
|
||||||
|
{batch, conn} = Client.command(conn, ["SPOP", "bag", "5"])
|
||||||
|
assert length(batch) == 2
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "bag"])
|
||||||
|
|
||||||
|
assert {nil, conn} = Client.command(conn, ["SPOP", "bag"])
|
||||||
|
assert {[], conn} = Client.command(conn, ["SPOP", "bag", "3"])
|
||||||
|
|
||||||
|
assert {5, conn} = Client.command(conn, ["SADD", "bag", "a", "b", "c", "d", "e"])
|
||||||
|
assert {[], conn} = Client.command(conn, ["SPOP", "bag", "0"])
|
||||||
|
assert {5, conn} = Client.command(conn, ["SCARD", "bag"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is out of range, must be positive"}, _conn} =
|
||||||
|
Client.command(conn, ["SPOP", "bag", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SRANDMEMBER repeats members for a negative count", %{conn: conn} do
|
||||||
|
assert {2, conn} = Client.command(conn, ["SADD", "colors", "red", "green"])
|
||||||
|
|
||||||
|
{single, conn} = Client.command(conn, ["SRANDMEMBER", "colors"])
|
||||||
|
assert single in ["red", "green"]
|
||||||
|
|
||||||
|
{repeats, conn} = Client.command(conn, ["SRANDMEMBER", "colors", "-5"])
|
||||||
|
assert length(repeats) == 5
|
||||||
|
assert Enum.all?(repeats, &(&1 in ["red", "green"]))
|
||||||
|
|
||||||
|
{capped, conn} = Client.command(conn, ["SRANDMEMBER", "colors", "5"])
|
||||||
|
assert Enum.sort(capped) == ["green", "red"]
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["SCARD", "colors"])
|
||||||
|
assert {nil, conn} = Client.command(conn, ["SRANDMEMBER", "nothing"])
|
||||||
|
assert {[], _conn} = Client.command(conn, ["SRANDMEMBER", "nothing", "3"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SMOVE moves a member and refuses a destination of the wrong type", %{conn: conn} do
|
||||||
|
assert {2, conn} = Client.command(conn, ["SADD", "source", "x", "y"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["SADD", "target", "z"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["SMOVE", "source", "target", "x"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["SMOVE", "source", "target", "ghost"])
|
||||||
|
|
||||||
|
{members, conn} = Client.command(conn, ["SMEMBERS", "target"])
|
||||||
|
assert Enum.sort(members) == ["x", "z"]
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "notaset", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["SMOVE", "source", "notaset", "y"])
|
||||||
|
|
||||||
|
{left, _conn} = Client.command(conn, ["SMEMBERS", "source"])
|
||||||
|
assert left == ["y"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LREM counts from the tail for a negative count", %{conn: conn} do
|
||||||
|
assert {5, conn} = Client.command(conn, ["RPUSH", "log", "a", "b", "a", "c", "a"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["LREM", "log", "-2", "a"])
|
||||||
|
|
||||||
|
{rest, conn} = Client.command(conn, ["LRANGE", "log", "0", "-1"])
|
||||||
|
assert rest == ["a", "b", "c"]
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LREM", "log", "0", "a"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["LREM", "log", "3", "zz"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["LREM", "absent", "0", "a"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, _conn} =
|
||||||
|
Client.command(conn, ["LREM", "log", "many", "a"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LSET reports missing keys and out of range indexes", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR no such key"}, conn} =
|
||||||
|
Client.command(conn, ["LSET", "nolist", "0", "v"])
|
||||||
|
|
||||||
|
assert {3, conn} = Client.command(conn, ["RPUSH", "items", "a", "b", "c"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["LSET", "items", "-1", "z"])
|
||||||
|
|
||||||
|
{items, conn} = Client.command(conn, ["LRANGE", "items", "0", "-1"])
|
||||||
|
assert items == ["a", "b", "z"]
|
||||||
|
|
||||||
|
assert {{:error, "ERR index out of range"}, conn} =
|
||||||
|
Client.command(conn, ["LSET", "items", "9", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR index out of range"}, _conn} =
|
||||||
|
Client.command(conn, ["LSET", "items", "-9", "v"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LTRIM keeps an inclusive window and drops an empty list", %{conn: conn} do
|
||||||
|
assert {5, conn} = Client.command(conn, ["RPUSH", "queue", "a", "b", "c", "d", "e"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["LTRIM", "queue", "1", "-2"])
|
||||||
|
|
||||||
|
{kept, conn} = Client.command(conn, ["LRANGE", "queue", "0", "-1"])
|
||||||
|
assert kept == ["b", "c", "d"]
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["LTRIM", "queue", "5", "10"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "queue"])
|
||||||
|
assert {{:simple, "OK"}, _conn} = Client.command(conn, ["LTRIM", "absent", "0", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LINSERT places an element around a pivot", %{conn: conn} do
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "plan", "one", "three"])
|
||||||
|
assert {3, conn} = Client.command(conn, ["LINSERT", "plan", "BEFORE", "three", "two"])
|
||||||
|
assert {4, conn} = Client.command(conn, ["LINSERT", "plan", "AFTER", "three", "four"])
|
||||||
|
|
||||||
|
{plan, conn} = Client.command(conn, ["LRANGE", "plan", "0", "-1"])
|
||||||
|
assert plan == ["one", "two", "three", "four"]
|
||||||
|
|
||||||
|
assert {-1, conn} = Client.command(conn, ["LINSERT", "plan", "BEFORE", "ghost", "x"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["LINSERT", "absent", "BEFORE", "a", "b"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["LINSERT", "plan", "SIDEWAYS", "one", "x"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "RPOPLPUSH moves, rotates and protects the destination", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["RPUSH", "source", "a", "b", "c"])
|
||||||
|
assert {"c", conn} = Client.command(conn, ["RPOPLPUSH", "source", "target"])
|
||||||
|
|
||||||
|
{target, conn} = Client.command(conn, ["LRANGE", "target", "0", "-1"])
|
||||||
|
assert target == ["c"]
|
||||||
|
|
||||||
|
assert {"b", conn} = Client.command(conn, ["RPOPLPUSH", "source", "source"])
|
||||||
|
{rotated, conn} = Client.command(conn, ["LRANGE", "source", "0", "-1"])
|
||||||
|
assert rotated == ["b", "a"]
|
||||||
|
|
||||||
|
assert {nil, conn} = Client.command(conn, ["RPOPLPUSH", "absent", "target"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "text", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["RPOPLPUSH", "source", "text"])
|
||||||
|
|
||||||
|
{intact, _conn} = Client.command(conn, ["LRANGE", "source", "0", "-1"])
|
||||||
|
assert intact == ["b", "a"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LPOS honours RANK, COUNT and MAXLEN", %{conn: conn} do
|
||||||
|
assert {8, conn} =
|
||||||
|
Client.command(conn, ["RPUSH", "mylist", "a", "b", "c", "1", "2", "3", "c", "c"])
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["LPOS", "mylist", "c"])
|
||||||
|
assert {6, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "2"])
|
||||||
|
assert {7, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "-1"])
|
||||||
|
assert {6, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "-2"])
|
||||||
|
|
||||||
|
{all, conn} = Client.command(conn, ["LPOS", "mylist", "c", "COUNT", "0"])
|
||||||
|
assert all == [2, 6, 7]
|
||||||
|
|
||||||
|
{two, conn} = Client.command(conn, ["LPOS", "mylist", "c", "COUNT", "2"])
|
||||||
|
assert two == [2, 6]
|
||||||
|
|
||||||
|
{backwards, conn} = Client.command(conn, ["LPOS", "mylist", "c", "RANK", "-1", "COUNT", "0"])
|
||||||
|
assert backwards == [7, 6, 2]
|
||||||
|
|
||||||
|
assert {nil, conn} = Client.command(conn, ["LPOS", "mylist", "ghost"])
|
||||||
|
{empty, conn} = Client.command(conn, ["LPOS", "mylist", "ghost", "COUNT", "0"])
|
||||||
|
assert empty == []
|
||||||
|
|
||||||
|
assert {nil, conn} = Client.command(conn, ["LPOS", "mylist", "c", "MAXLEN", "2"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["LPOS", "mylist", "c", "MAXLEN", "3"])
|
||||||
|
assert {nil, conn} = Client.command(conn, ["LPOS", "absent", "c"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR RANK can't be zero." <> _}, conn} =
|
||||||
|
Client.command(conn, ["LPOS", "mylist", "c", "RANK", "0"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR COUNT can't be negative"}, conn} =
|
||||||
|
Client.command(conn, ["LPOS", "mylist", "c", "COUNT", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR MAXLEN can't be negative"}, conn} =
|
||||||
|
Client.command(conn, ["LPOS", "mylist", "c", "MAXLEN", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["LPOS", "mylist", "c", "NEAR", "1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "ZCOUNT handles exclusive and infinite bounds", %{conn: conn} do
|
||||||
|
assert {4, conn} =
|
||||||
|
Client.command(conn, ["ZADD", "prices", "10", "a", "20", "b", "30", "c", "40", "d"])
|
||||||
|
|
||||||
|
assert {4, conn} = Client.command(conn, ["ZCOUNT", "prices", "-inf", "+inf"])
|
||||||
|
assert {3, conn} = Client.command(conn, ["ZCOUNT", "prices", "20", "40"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["ZCOUNT", "prices", "(20", "40"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["ZCOUNT", "prices", "(20", "(40"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["ZCOUNT", "prices", "-inf", "20"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["ZCOUNT", "absent", "-inf", "+inf"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR min or max is not a float"}, _conn} =
|
||||||
|
Client.command(conn, ["ZCOUNT", "prices", "abc", "10"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "ZREVRANGE, ZREVRANK and ZMSCORE", %{conn: conn} do
|
||||||
|
assert {3, conn} = Client.command(conn, ["ZADD", "board", "1", "a", "2", "b", "3.5", "c"])
|
||||||
|
|
||||||
|
{desc, conn} = Client.command(conn, ["ZREVRANGE", "board", "0", "-1"])
|
||||||
|
assert desc == ["c", "b", "a"]
|
||||||
|
|
||||||
|
{top, conn} = Client.command(conn, ["ZREVRANGE", "board", "0", "0"])
|
||||||
|
assert top == ["c"]
|
||||||
|
|
||||||
|
{tail, conn} = Client.command(conn, ["ZREVRANGE", "board", "-2", "-1"])
|
||||||
|
assert tail == ["b", "a"]
|
||||||
|
|
||||||
|
{scored, conn} = Client.command(conn, ["ZREVRANGE", "board", "0", "-1", "WITHSCORES"])
|
||||||
|
assert scored == ["c", "3.5", "b", "2", "a", "1"]
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["ZREVRANGE", "board", "0", "-1", "WITHVALUES"])
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["ZREVRANK", "board", "c"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["ZREVRANK", "board", "a"])
|
||||||
|
assert {nil, conn} = Client.command(conn, ["ZREVRANK", "board", "ghost"])
|
||||||
|
|
||||||
|
{scores, conn} = Client.command(conn, ["ZMSCORE", "board", "a", "ghost", "c"])
|
||||||
|
assert scores == ["1", nil, "3.5"]
|
||||||
|
|
||||||
|
{absent, _conn} = Client.command(conn, ["ZMSCORE", "nozset", "a"])
|
||||||
|
assert absent == [nil]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "RENAME and RENAMENX", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "old", "payload"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIRE", "old", "100"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["RENAME", "old", "fresh"])
|
||||||
|
assert {"payload", conn} = Client.command(conn, ["GET", "fresh"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "old"])
|
||||||
|
|
||||||
|
{ttl, conn} = Client.command(conn, ["TTL", "fresh"])
|
||||||
|
assert ttl > 0
|
||||||
|
|
||||||
|
assert {{:error, "ERR no such key"}, conn} = Client.command(conn, ["RENAME", "old", "other"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "taken", "t"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["RENAMENX", "fresh", "taken"])
|
||||||
|
assert {"payload", conn} = Client.command(conn, ["GET", "fresh"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["RENAMENX", "fresh", "free"])
|
||||||
|
assert {"payload", conn} = Client.command(conn, ["GET", "free"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR no such key"}, _conn} = Client.command(conn, ["RENAMENX", "ghost", "x"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "COPY with REPLACE and DB", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "origin", "payload"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["COPY", "origin", "clone"])
|
||||||
|
assert {"payload", conn} = Client.command(conn, ["GET", "clone"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["COPY", "origin", "clone"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "origin", "changed"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["COPY", "origin", "clone", "REPLACE"])
|
||||||
|
assert {"changed", conn} = Client.command(conn, ["GET", "clone"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["COPY", "origin", "sameDb", "DB", "0"])
|
||||||
|
assert {"changed", conn} = Client.command(conn, ["GET", "sameDb"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR DB index is out of range"}, conn} =
|
||||||
|
Client.command(conn, ["COPY", "origin", "elsewhere", "DB", "3"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR source and destination objects are the same"}, conn} =
|
||||||
|
Client.command(conn, ["COPY", "origin", "origin"])
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["COPY", "ghost", "anywhere"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, _conn} =
|
||||||
|
Client.command(conn, ["COPY", "origin", "clone", "FORCE"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "TOUCH and UNLINK count existing keys", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MSET", "t1", "1", "t2", "2"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["TOUCH", "t1", "t2", "t3"])
|
||||||
|
assert {2, conn} = Client.command(conn, ["UNLINK", "t1", "t2", "t3"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "t1"])
|
||||||
|
assert {0, _conn} = Client.command(conn, ["TOUCH", "t1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "EXPIREAT, EXPIRETIME and PEXPIRETIME", %{conn: conn} do
|
||||||
|
future = System.system_time(:second) + 100
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "session", "v"])
|
||||||
|
assert {-1, conn} = Client.command(conn, ["EXPIRETIME", "session"])
|
||||||
|
assert {-1, conn} = Client.command(conn, ["PEXPIRETIME", "session"])
|
||||||
|
assert {-2, conn} = Client.command(conn, ["EXPIRETIME", "ghost"])
|
||||||
|
assert {-2, conn} = Client.command(conn, ["PEXPIRETIME", "ghost"])
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXPIREAT", "ghost", Integer.to_string(future)])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIREAT", "session", Integer.to_string(future)])
|
||||||
|
assert {^future, conn} = Client.command(conn, ["EXPIRETIME", "session"])
|
||||||
|
|
||||||
|
{millis, conn} = Client.command(conn, ["PEXPIRETIME", "session"])
|
||||||
|
assert millis == future * 1000
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "doomed", "v"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIREAT", "doomed", "1000"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "doomed"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, _conn} =
|
||||||
|
Client.command(conn, ["EXPIREAT", "session", "soon"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "EXPIREAT respects NX, XX, GT and LT", %{conn: conn} do
|
||||||
|
base = System.system_time(:second) + 100
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "lease", "v"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "XX"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "GT"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "NX"])
|
||||||
|
assert {^base, conn} = Client.command(conn, ["EXPIRETIME", "lease"])
|
||||||
|
|
||||||
|
assert {0, conn} =
|
||||||
|
Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base + 50), "NX"])
|
||||||
|
|
||||||
|
later = base + 50
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(later), "GT"])
|
||||||
|
assert {^later, conn} = Client.command(conn, ["EXPIRETIME", "lease"])
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "GT"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "LT"])
|
||||||
|
assert {^base, conn} = Client.command(conn, ["EXPIRETIME", "lease"])
|
||||||
|
assert {1, conn} = Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(later), "XX"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR NX and XX, GT or LT options at the same time are not compatible"}, conn} =
|
||||||
|
Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "NX", "XX"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Unsupported option SOON"}, _conn} =
|
||||||
|
Client.command(conn, ["EXPIREAT", "lease", Integer.to_string(base), "soon"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "OBJECT reports encoding, refcount, idletime and help", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "number", "12345"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "shared", "42"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "padded", "007"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "short", "hello"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["SET", "long", String.duplicate("x", 60)])
|
||||||
|
|
||||||
|
assert {"int", conn} = Client.command(conn, ["OBJECT", "ENCODING", "number"])
|
||||||
|
assert {"embstr", conn} = Client.command(conn, ["OBJECT", "ENCODING", "padded"])
|
||||||
|
assert {"embstr", conn} = Client.command(conn, ["OBJECT", "ENCODING", "short"])
|
||||||
|
assert {"raw", conn} = Client.command(conn, ["OBJECT", "ENCODING", "long"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "smallList", "a"])
|
||||||
|
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "smallList"])
|
||||||
|
|
||||||
|
bigList = ["RPUSH", "bigList"] ++ Enum.map(1..200, &Integer.to_string/1)
|
||||||
|
assert {200, conn} = Client.command(conn, bigList)
|
||||||
|
assert {"quicklist", conn} = Client.command(conn, ["OBJECT", "ENCODING", "bigList"])
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["SADD", "ints", "1", "2"])
|
||||||
|
assert {"intset", conn} = Client.command(conn, ["OBJECT", "ENCODING", "ints"])
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["SADD", "words", "alpha", "beta"])
|
||||||
|
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "words"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["HSET", "hash", "field", "value"])
|
||||||
|
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "hash"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["ZADD", "zset", "1", "a"])
|
||||||
|
assert {"listpack", conn} = Client.command(conn, ["OBJECT", "ENCODING", "zset"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["OBJECT", "REFCOUNT", "short"])
|
||||||
|
assert {2_147_483_647, conn} = Client.command(conn, ["OBJECT", "REFCOUNT", "shared"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["OBJECT", "IDLETIME", "short"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR no such key"}, conn} =
|
||||||
|
Client.command(conn, ["OBJECT", "ENCODING", "ghost"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR no such key"}, conn} =
|
||||||
|
Client.command(conn, ["OBJECT", "REFCOUNT", "ghost"])
|
||||||
|
|
||||||
|
{help, conn} = Client.command(conn, ["OBJECT", "HELP"])
|
||||||
|
assert Enum.any?(help, &match?({:simple, "ENCODING <key>"}, &1))
|
||||||
|
|
||||||
|
assert {{:error, "ERR Unknown subcommand" <> _}, _conn} =
|
||||||
|
Client.command(conn, ["OBJECT", "SIZE", "short"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "wrong arity is reported per command", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'strlen' command"}, conn} =
|
||||||
|
Client.command(conn, ["STRLEN"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'lset' command"}, conn} =
|
||||||
|
Client.command(conn, ["LSET", "key"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'hmget' command"}, conn} =
|
||||||
|
Client.command(conn, ["HMGET", "key"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'copy' command"}, conn} =
|
||||||
|
Client.command(conn, ["COPY", "key"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'unlink' command"}, conn} =
|
||||||
|
Client.command(conn, ["UNLINK"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'expiretime' command"}, _conn} =
|
||||||
|
Client.command(conn, ["EXPIRETIME", "a", "b"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "wrong types and non integer arguments are rejected", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "listkey", "x"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "stringkey", "x"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["STRLEN", "listkey"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["GETRANGE", "listkey", "0", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["LPOS", "stringkey", "x"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["ZCOUNT", "stringkey", "-inf", "+inf"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["ZREVRANGE", "stringkey", "0", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["ZMSCORE", "stringkey", "a"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["LTRIM", "stringkey", "0", "-1"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} = Client.command(conn, ["SPOP", "stringkey"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["SMISMEMBER", "stringkey", "a"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
|
||||||
|
Client.command(conn, ["GETRANGE", "stringkey", "a", "1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
|
||||||
|
Client.command(conn, ["SETRANGE", "stringkey", "here", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, _conn} =
|
||||||
|
Client.command(conn, ["SRANDMEMBER", "stringkey", "few"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "keyspace commands are queued and replayed inside MULTI", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "counter", "1"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
|
||||||
|
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["STRLEN", "counter"])
|
||||||
|
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["COPY", "counter", "backup"])
|
||||||
|
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["TOUCH", "counter", "backup"])
|
||||||
|
|
||||||
|
{replies, conn} = Client.command(conn, ["EXEC"])
|
||||||
|
assert replies == [1, 1, 2]
|
||||||
|
|
||||||
|
assert {"1", _conn} = Client.command(conn, ["GET", "backup"])
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
defmodule Excache.MemoryTest do
|
||||||
|
use ExUnit.Case, async: false
|
||||||
|
|
||||||
|
alias Excache.Memory
|
||||||
|
alias Excache.SkipList
|
||||||
|
alias Excache.Store
|
||||||
|
alias Excache.Stream, as: CacheStream
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Store.flushdb()
|
||||||
|
on_exit(&Store.flushdb/0)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "entry overhead matches the real cost of an ETS insert" do
|
||||||
|
entries = for index <- 1..2000, do: {"probe:#{index}", :string, "value-#{index}", nil}
|
||||||
|
|
||||||
|
table = :ets.new(:memoryProbe, [:set, :public])
|
||||||
|
before = :ets.info(table, :memory)
|
||||||
|
Enum.each(entries, &:ets.insert(table, &1))
|
||||||
|
measured = (:ets.info(table, :memory) - before) * Memory.wordSize()
|
||||||
|
:ets.delete(table)
|
||||||
|
|
||||||
|
estimated = Memory.usedMemory(entries)
|
||||||
|
|
||||||
|
assert measured > 0
|
||||||
|
assert_in_delta estimated / measured, 1.0, 0.02
|
||||||
|
end
|
||||||
|
|
||||||
|
test "reference counted binaries are counted beyond the flat term size" do
|
||||||
|
value = :binary.copy("x", 4096)
|
||||||
|
flat = :erts_debug.size(value) * Memory.wordSize()
|
||||||
|
|
||||||
|
assert flat < 200
|
||||||
|
assert Memory.estimateValue(:string, value) == flat + 4096
|
||||||
|
|
||||||
|
nested = [:binary.copy("a", 1024), :binary.copy("b", 1024)]
|
||||||
|
assert Memory.estimateValue(:list, nested) >= 2048
|
||||||
|
|
||||||
|
heapBinary = :binary.copy("y", 64)
|
||||||
|
|
||||||
|
assert Memory.estimateValue(:string, heapBinary) ==
|
||||||
|
:erts_debug.size(heapBinary) * Memory.wordSize()
|
||||||
|
end
|
||||||
|
|
||||||
|
test "estimates grow with the element count of every aggregate type" do
|
||||||
|
pairs = [
|
||||||
|
{:list, listValue(2), listValue(40)},
|
||||||
|
{:hash, hashValue(2), hashValue(40)},
|
||||||
|
{:set, setValue(2), setValue(40)},
|
||||||
|
{:zset, zsetValue(2), zsetValue(40)},
|
||||||
|
{:stream, streamValue(2), streamValue(40)}
|
||||||
|
]
|
||||||
|
|
||||||
|
for {kind, small, large} <- pairs do
|
||||||
|
assert Memory.estimateValue(kind, small) > 0, "#{kind} small estimate is not positive"
|
||||||
|
|
||||||
|
assert Memory.estimateValue(kind, large) > Memory.estimateValue(kind, small),
|
||||||
|
"#{kind} estimate did not grow with the element count"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "element counts and payload bytes are reported per type" do
|
||||||
|
assert Memory.elementCount(:string, "abc") == 1
|
||||||
|
assert Memory.payloadBytes(:string, "abc") == 3
|
||||||
|
|
||||||
|
assert Memory.elementCount(:list, ["a", "bb"]) == 2
|
||||||
|
assert Memory.payloadBytes(:list, ["a", "bb"]) == 3
|
||||||
|
|
||||||
|
hash = %{"f1" => "v1", "f2" => "vvv2"}
|
||||||
|
assert Memory.elementCount(:hash, hash) == 2
|
||||||
|
assert Memory.payloadBytes(:hash, hash) == 2 + 2 + 2 + 4
|
||||||
|
|
||||||
|
set = MapSet.new(["a", "bb", "ccc"])
|
||||||
|
assert Memory.elementCount(:set, set) == 3
|
||||||
|
assert Memory.payloadBytes(:set, set) == 6
|
||||||
|
|
||||||
|
zset = zsetValue(2)
|
||||||
|
assert Memory.elementCount(:zset, zset) == 2
|
||||||
|
assert Memory.payloadBytes(:zset, zset) == 2 * (byte_size("member1") + Memory.wordSize())
|
||||||
|
|
||||||
|
stream = streamValue(2)
|
||||||
|
assert Memory.elementCount(:stream, stream) == 2
|
||||||
|
|
||||||
|
assert Memory.payloadBytes(:stream, stream) ==
|
||||||
|
2 * (2 * Memory.wordSize() + byte_size("field") + byte_size("value1"))
|
||||||
|
end
|
||||||
|
|
||||||
|
test "usedMemory sums every entry and usageByType splits them by kind" do
|
||||||
|
entries = [
|
||||||
|
{"s", :string, "value", nil},
|
||||||
|
{"l", :list, listValue(3), nil},
|
||||||
|
{"h", :hash, hashValue(3), nil},
|
||||||
|
{"st", :set, setValue(3), nil},
|
||||||
|
{"z", :zset, zsetValue(3), nil},
|
||||||
|
{"x", :stream, streamValue(3), nil}
|
||||||
|
]
|
||||||
|
|
||||||
|
assert Memory.usedMemory(entries) ==
|
||||||
|
Enum.reduce(entries, 0, fn entry, total -> total + Memory.entryBytes(entry) end)
|
||||||
|
|
||||||
|
byType = Memory.usageByType(entries)
|
||||||
|
assert Map.keys(byType) |> Enum.sort() == Enum.sort(Memory.kinds())
|
||||||
|
assert Enum.all?(byType, fn {_kind, usage} -> usage.keys == 1 end)
|
||||||
|
|
||||||
|
assert byType.string.bytes == Memory.entryBytes({"s", :string, "value", nil})
|
||||||
|
|
||||||
|
assert Enum.sum(Enum.map(byType, fn {_kind, usage} -> usage.bytes end)) ==
|
||||||
|
Memory.usedMemory(entries)
|
||||||
|
|
||||||
|
withoutStreams = Enum.reject(entries, fn {_key, kind, _value, _ea} -> kind == :stream end)
|
||||||
|
assert Memory.usageByType(withoutStreams).stream == %{keys: 0, bytes: 0}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "stats separate overhead from dataset and point at the largest key" do
|
||||||
|
entries = [
|
||||||
|
{"tiny", :string, "a", nil},
|
||||||
|
{"huge", :string, :binary.copy("z", 8192), 1_700_000_000_000},
|
||||||
|
{"mid", :list, listValue(10), nil}
|
||||||
|
]
|
||||||
|
|
||||||
|
stats = Memory.stats(entries)
|
||||||
|
|
||||||
|
assert stats.keys == 3
|
||||||
|
assert stats.volatileKeys == 1
|
||||||
|
assert stats.usedMemory == Memory.usedMemory(entries)
|
||||||
|
assert stats.datasetBytes == stats.usedMemory - stats.overheadBytes
|
||||||
|
assert stats.overheadBytes == 3 * Memory.entryOverhead() + overheadKeyBytes(entries)
|
||||||
|
assert stats.bytesPerKey == div(stats.usedMemory, 3)
|
||||||
|
assert {"huge", huge} = stats.largestKey
|
||||||
|
assert huge >= 8192
|
||||||
|
assert Memory.stats([]).largestKey == nil
|
||||||
|
assert Memory.stats([]).bytesPerKey == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "describe reports the volatile flag and separates payload from overhead" do
|
||||||
|
value = :binary.copy("p", 512)
|
||||||
|
described = Memory.describe("session:1", {:string, value, 1_700_000_000_000})
|
||||||
|
|
||||||
|
assert described.kind == :string
|
||||||
|
assert described.volatile
|
||||||
|
assert described.payloadBytes == 512
|
||||||
|
assert described.bytes == described.keyBytes + described.valueBytes + Memory.entryOverhead()
|
||||||
|
assert described.overheadBytes == described.bytes - 512
|
||||||
|
assert Memory.describe("k", {:string, "v", nil}).volatile == false
|
||||||
|
end
|
||||||
|
|
||||||
|
test "usage reads a live key and reports nil for a missing one" do
|
||||||
|
:ok = Store.set("alpha", "value", [])
|
||||||
|
{:ok, 3} = Store.rpush("beta", ["a", "b", "c"])
|
||||||
|
|
||||||
|
assert Memory.usage("alpha") == Memory.estimateEntry("alpha", Store.watch_snapshot("alpha"))
|
||||||
|
assert Memory.usage("ghost") == nil
|
||||||
|
assert Memory.describeKey("ghost") == nil
|
||||||
|
assert Memory.describeKey("beta").kind == :list
|
||||||
|
assert Memory.describeKey("beta").elements == 3
|
||||||
|
|
||||||
|
assert Memory.usedMemory() == Memory.usage("alpha") + Memory.usage("beta")
|
||||||
|
assert Memory.usageByType().list.keys == 1
|
||||||
|
assert Memory.stats().keys == 2
|
||||||
|
end
|
||||||
|
|
||||||
|
test "keyBytes grows with the key length" do
|
||||||
|
assert Memory.keyBytes(String.duplicate("k", 200)) > Memory.keyBytes("k")
|
||||||
|
assert Memory.entryOverhead() == 10 * Memory.wordSize()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp overheadKeyBytes(entries) do
|
||||||
|
Enum.reduce(entries, 0, fn {key, _kind, _value, _ea}, total ->
|
||||||
|
total + Memory.keyBytes(key)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp listValue(count), do: Enum.map(1..count, &"item#{&1}")
|
||||||
|
|
||||||
|
defp hashValue(count), do: Map.new(1..count, fn index -> {"field#{index}", "value#{index}"} end)
|
||||||
|
|
||||||
|
defp setValue(count), do: MapSet.new(1..count, &"member#{&1}")
|
||||||
|
|
||||||
|
defp zsetValue(count) do
|
||||||
|
Enum.reduce(1..count, SkipList.new(), fn index, zset ->
|
||||||
|
SkipList.insert(zset, "member#{index}", index * 1.0)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp streamValue(count) do
|
||||||
|
Enum.reduce(1..count, CacheStream.new(), fn index, stream ->
|
||||||
|
{:ok, updated, _id} =
|
||||||
|
CacheStream.add(stream, {:full, index, 0}, ["field", "value#{index}"], 0)
|
||||||
|
|
||||||
|
updated
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
defmodule Excache.ServerCommandTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Config.reset()
|
||||||
|
Runner.resetStats()
|
||||||
|
on_exit(&Config.reset/0)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "CONFIG GET answers with the live values and follows the pattern", %{conn: conn} do
|
||||||
|
assert {["maxmemory", "0"], conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
|
||||||
|
|
||||||
|
assert {values, conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory*"])
|
||||||
|
|
||||||
|
assert values == [
|
||||||
|
"maxmemory",
|
||||||
|
"0",
|
||||||
|
"maxmemory-policy",
|
||||||
|
"noeviction",
|
||||||
|
"maxmemory-samples",
|
||||||
|
"5"
|
||||||
|
]
|
||||||
|
|
||||||
|
assert {[], conn} = Client.command(conn, ["CONFIG", "GET", "nothing-like-this"])
|
||||||
|
|
||||||
|
assert {merged, conn} = Client.command(conn, ["CONFIG", "GET", "lfu-log-factor", "maxmemory"])
|
||||||
|
assert merged == ["lfu-log-factor", "10", "maxmemory", "0"]
|
||||||
|
|
||||||
|
assert {deduped, conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory", "maxmemory"])
|
||||||
|
assert deduped == ["maxmemory", "0"]
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'config|get' command"}, _conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "GET"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "CONFIG SET stores what CONFIG GET reads back", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "1mb"])
|
||||||
|
assert {["maxmemory", "1048576"], conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
|
||||||
|
assert Config.maxmemory() == 1_048_576
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory-policy", "allkeys-lru"])
|
||||||
|
|
||||||
|
assert {["maxmemory-policy", "allkeys-lru"], conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "GET", "maxmemory-policy"])
|
||||||
|
|
||||||
|
assert Config.policy() == :allkeysLru
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, [
|
||||||
|
"CONFIG",
|
||||||
|
"SET",
|
||||||
|
"maxmemory",
|
||||||
|
"0",
|
||||||
|
"maxmemory-samples",
|
||||||
|
"16"
|
||||||
|
])
|
||||||
|
|
||||||
|
assert {["maxmemory", "0", "maxmemory-samples", "16"], _conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "GET", "maxmemory", "maxmemory-samples"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "CONFIG SET reports every way it can be wrong", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR CONFIG SET failed - argument must be a memory value" <> _}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory", "plenty"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR CONFIG SET failed - argument must be one of the following:" <> _}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory-policy", "lru"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Unknown option or number of arguments for CONFIG SET - 'nosuch'"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "nosuch", "1"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR CONFIG SET failed - can't set immutable config 'port'"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "port", "6380"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'config|set' command"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR wrong number of arguments for 'config|set' command"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory", "1kb", "lfu-log-factor"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Unknown subcommand" <> _}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "REWRITE"])
|
||||||
|
|
||||||
|
assert {["maxmemory", "0"], _conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a rejected pair leaves the whole CONFIG SET unapplied", %{conn: conn} do
|
||||||
|
assert {{:error, _reason}, conn} =
|
||||||
|
Client.command(conn, [
|
||||||
|
"CONFIG",
|
||||||
|
"SET",
|
||||||
|
"maxmemory",
|
||||||
|
"2mb",
|
||||||
|
"maxmemory-policy",
|
||||||
|
"nope"
|
||||||
|
])
|
||||||
|
|
||||||
|
assert {["maxmemory", "0", "maxmemory-policy", "noeviction"], _conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "GET", "maxmemory", "maxmemory-policy"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "MEMORY USAGE reports the real cost of a key", %{conn: conn} do
|
||||||
|
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "small", "x"])
|
||||||
|
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "large", String.duplicate("x", 4096)])
|
||||||
|
|
||||||
|
assert {small, conn} = Client.command(conn, ["MEMORY", "USAGE", "small"])
|
||||||
|
assert {large, conn} = Client.command(conn, ["MEMORY", "USAGE", "large"])
|
||||||
|
|
||||||
|
assert small > 0
|
||||||
|
assert large > small + 4000
|
||||||
|
assert large == Excache.Memory.usage("large")
|
||||||
|
|
||||||
|
assert {nil, conn} = Client.command(conn, ["MEMORY", "USAGE", "ghost"])
|
||||||
|
assert {^large, conn} = Client.command(conn, ["MEMORY", "USAGE", "large", "SAMPLES", "5"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR value is not an integer or out of range"}, conn} =
|
||||||
|
Client.command(conn, ["MEMORY", "USAGE", "large", "SAMPLES", "many"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR syntax error"}, conn} =
|
||||||
|
Client.command(conn, ["MEMORY", "USAGE", "large", "COUNT", "5"])
|
||||||
|
|
||||||
|
assert {{:error, "ERR Unknown subcommand" <> _}, _conn} =
|
||||||
|
Client.command(conn, ["MEMORY", "USAGE"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "MEMORY STATS adds up to the same numbers the store reports", %{conn: conn} do
|
||||||
|
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "plain", String.duplicate("a", 200)])
|
||||||
|
{2, conn} = Client.command(conn, ["RPUSH", "queue", "one", "two"])
|
||||||
|
{1, conn} = Client.command(conn, ["SADD", "tags", "red"])
|
||||||
|
{1, conn} = Client.command(conn, ["EXPIRE", "plain", "600"])
|
||||||
|
|
||||||
|
{stats, conn} = stats(conn)
|
||||||
|
|
||||||
|
assert stats["keys.count"] == 3
|
||||||
|
assert stats["keys.volatile"] == 1
|
||||||
|
assert stats["type.string.keys"] == 1
|
||||||
|
assert stats["type.list.keys"] == 1
|
||||||
|
assert stats["type.set.keys"] == 1
|
||||||
|
assert stats["type.hash.keys"] == 0
|
||||||
|
assert stats["maxmemory"] == 0
|
||||||
|
assert stats["maxmemory.policy"] == "noeviction"
|
||||||
|
assert stats["maxmemory.samples"] == 5
|
||||||
|
assert stats["evicted.keys"] == 0
|
||||||
|
assert stats["oom.rejections"] == 0
|
||||||
|
assert stats["largest.key"] == "plain"
|
||||||
|
|
||||||
|
assert stats["total.allocated"] ==
|
||||||
|
stats["type.string.bytes"] + stats["type.list.bytes"] + stats["type.set.bytes"]
|
||||||
|
|
||||||
|
assert stats["total.allocated"] == Excache.Memory.usedMemory()
|
||||||
|
assert stats["dataset.bytes"] + stats["overhead.total"] == stats["total.allocated"]
|
||||||
|
assert stats["keys.bytes-per-key"] == div(stats["total.allocated"], 3)
|
||||||
|
assert stats["largest.key.bytes"] == Excache.Memory.usage("plain")
|
||||||
|
|
||||||
|
{_, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "4mb"])
|
||||||
|
{updated, _conn} = stats(conn)
|
||||||
|
assert updated["maxmemory"] == 4_194_304
|
||||||
|
end
|
||||||
|
|
||||||
|
test "MEMORY DOCTOR describes the store it is actually looking at", %{conn: conn} do
|
||||||
|
assert {"Excache holds no keys, there is nothing to report.", conn} =
|
||||||
|
Client.command(conn, ["MEMORY", "DOCTOR"])
|
||||||
|
|
||||||
|
{{:simple, "OK"}, conn} = Client.command(conn, ["SET", "hog", String.duplicate("z", 20_000)])
|
||||||
|
|
||||||
|
conn =
|
||||||
|
Enum.reduce(1..5, conn, fn index, acc ->
|
||||||
|
{{:simple, "OK"}, acc} = Client.command(acc, ["SET", "tiny:#{index}", "v"])
|
||||||
|
acc
|
||||||
|
end)
|
||||||
|
|
||||||
|
{report, conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
|
||||||
|
|
||||||
|
assert report =~ "6 keys"
|
||||||
|
assert report =~ "no limit"
|
||||||
|
assert report =~ "maxmemory-policy noeviction"
|
||||||
|
assert report =~ "maxmemory is not set"
|
||||||
|
assert report =~ "key 'hog' holds"
|
||||||
|
assert report =~ "percent of everything stored"
|
||||||
|
|
||||||
|
{_, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory-policy", "volatile-ttl"])
|
||||||
|
{_, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "1mb"])
|
||||||
|
{volatile, conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
|
||||||
|
|
||||||
|
assert volatile =~ "1048576 byte limit"
|
||||||
|
assert volatile =~ "only evicts keys with a TTL and none of the 6 keys has one"
|
||||||
|
refute volatile =~ "maxmemory is not set"
|
||||||
|
|
||||||
|
{1, conn} = Client.command(conn, ["EXPIRE", "hog", "600"])
|
||||||
|
{withTtl, _conn} = Client.command(conn, ["MEMORY", "DOCTOR"])
|
||||||
|
refute withTtl =~ "only evicts keys with a TTL"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "MEMORY and CONFIG are known commands inside transactions", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["MULTI"])
|
||||||
|
|
||||||
|
assert {{:simple, "QUEUED"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory", "2kb"])
|
||||||
|
|
||||||
|
assert {{:simple, "QUEUED"}, conn} = Client.command(conn, ["CONFIG", "GET", "maxmemory"])
|
||||||
|
assert {[{:simple, "OK"}, ["maxmemory", "2048"]], _conn} = Client.command(conn, ["EXEC"])
|
||||||
|
assert Config.maxmemory() == 2048
|
||||||
|
end
|
||||||
|
|
||||||
|
defp stats(conn) do
|
||||||
|
{flat, conn} = Client.command(conn, ["MEMORY", "STATS"])
|
||||||
|
{flat |> Enum.chunk_every(2) |> Map.new(fn [name, value] -> {name, value} end), conn}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
defmodule Excache.StoreConsistencyTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@ttlMs 200
|
||||||
|
@holdMs 400
|
||||||
|
|
||||||
|
setup do
|
||||||
|
Tracker.clear(Tracker)
|
||||||
|
|
||||||
|
on_exit(fn ->
|
||||||
|
Config.reset()
|
||||||
|
Tracker.clear(Tracker)
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "RENAME must not drop the source when the destination write is refused" do
|
||||||
|
test "an expired source is reported instead of being silently swallowed" do
|
||||||
|
source = "cons:rename:src"
|
||||||
|
dest = keyInOtherShard("cons:rename:dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(source, "payload", px: @ttlMs)
|
||||||
|
|
||||||
|
assert {:error, :nokey} =
|
||||||
|
duringSuspendedShard(dest, fn -> Store.rename(source, dest) end)
|
||||||
|
|
||||||
|
assert Store.exists([dest]) == 0
|
||||||
|
assert Store.exists([source]) == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an existing destination is never reported as overwritten when it was not" do
|
||||||
|
source = "cons:rename:live-src"
|
||||||
|
dest = keyInOtherShard("cons:rename:live-dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(dest, "stale", [])
|
||||||
|
assert :ok = Store.set(source, "fresh", px: @ttlMs)
|
||||||
|
|
||||||
|
assert {:error, :nokey} =
|
||||||
|
duringSuspendedShard(dest, fn -> Store.rename(source, dest) end)
|
||||||
|
|
||||||
|
assert {:ok, "stale"} = Store.get(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a healthy rename still carries the value and the remaining ttl" do
|
||||||
|
source = "cons:rename:ok-src"
|
||||||
|
dest = keyInOtherShard("cons:rename:ok-dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(source, "payload", px: 60_000)
|
||||||
|
|
||||||
|
assert {:ok, "OK"} = Store.rename(source, dest)
|
||||||
|
assert {:ok, "payload"} = Store.get(dest)
|
||||||
|
assert Store.exists([source]) == 0
|
||||||
|
|
||||||
|
remaining = Store.ttl(dest, :seconds)
|
||||||
|
assert remaining > 0 and remaining <= 60
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "RENAMENX must not drop the source when the destination write is refused" do
|
||||||
|
test "an expired source is reported instead of counted as a rename" do
|
||||||
|
source = "cons:renamenx:src"
|
||||||
|
dest = keyInOtherShard("cons:renamenx:dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(source, "payload", px: @ttlMs)
|
||||||
|
|
||||||
|
assert {:error, :nokey} =
|
||||||
|
duringSuspendedShard(dest, fn -> Store.renamenx(source, dest) end)
|
||||||
|
|
||||||
|
assert Store.exists([dest]) == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an occupied destination still refuses the rename and keeps both keys" do
|
||||||
|
source = "cons:renamenx:busy-src"
|
||||||
|
dest = keyInOtherShard("cons:renamenx:busy-dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(source, "payload", [])
|
||||||
|
assert :ok = Store.set(dest, "taken", [])
|
||||||
|
|
||||||
|
assert {:ok, 0} = Store.renamenx(source, dest)
|
||||||
|
assert {:ok, "payload"} = Store.get(source)
|
||||||
|
assert {:ok, "taken"} = Store.get(dest)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "COPY must not destroy the destination when the write is refused" do
|
||||||
|
test "an expiring source leaves the existing destination intact" do
|
||||||
|
source = "cons:copy:src"
|
||||||
|
dest = keyInOtherShard("cons:copy:dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(dest, "keep-me", [])
|
||||||
|
assert :ok = Store.set(source, "payload", px: @ttlMs)
|
||||||
|
|
||||||
|
assert {:ok, 0} =
|
||||||
|
duringSuspendedShard(dest, fn -> Store.copy(source, dest, true) end)
|
||||||
|
|
||||||
|
assert {:ok, "keep-me"} = Store.get(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "replacing a destination of a different type still overwrites it" do
|
||||||
|
source = "cons:copy:swap-src"
|
||||||
|
dest = keyInOtherShard("cons:copy:swap-dst-", source)
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.rpush(dest, ["old"])
|
||||||
|
assert :ok = Store.set(source, "fresh", [])
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.copy(source, dest, true)
|
||||||
|
assert {:ok, "fresh"} = Store.get(dest)
|
||||||
|
assert Store.type(dest) == "string"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "copying onto an occupied destination without replace changes nothing" do
|
||||||
|
source = "cons:copy:keep-src"
|
||||||
|
dest = keyInOtherShard("cons:copy:keep-dst-", source)
|
||||||
|
|
||||||
|
assert :ok = Store.set(source, "fresh", [])
|
||||||
|
assert :ok = Store.set(dest, "untouched", [])
|
||||||
|
|
||||||
|
assert {:ok, 0} = Store.copy(source, dest, false)
|
||||||
|
assert {:ok, "untouched"} = Store.get(dest)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "SPOP must reach the log as the removal it actually performed" do
|
||||||
|
test "the replayed log rebuilds exactly the surviving members", %{conn: conn} do
|
||||||
|
key = "cons:pop:members"
|
||||||
|
members = Enum.map(1..20, &"member#{&1}")
|
||||||
|
|
||||||
|
assert {20, conn} = Client.command(conn, ["SADD", key | members])
|
||||||
|
|
||||||
|
{taken, conn} = Client.command(conn, ["SPOP", key, "5"])
|
||||||
|
assert length(taken) == 5
|
||||||
|
|
||||||
|
{single, conn} = Client.command(conn, ["SPOP", key])
|
||||||
|
assert single in members
|
||||||
|
|
||||||
|
{survivors, conn} = Client.command(conn, ["SMEMBERS", key])
|
||||||
|
assert length(survivors) == 14
|
||||||
|
|
||||||
|
log = flushedLog()
|
||||||
|
refute log =~ "SPOP"
|
||||||
|
assert log =~ "SREM"
|
||||||
|
|
||||||
|
conn = replayFromScratch(conn)
|
||||||
|
|
||||||
|
{replayed, _conn} = Client.command(conn, ["SMEMBERS", key])
|
||||||
|
assert Enum.sort(replayed) == Enum.sort(survivors)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "draining the whole set replays as the key being gone", %{conn: conn} do
|
||||||
|
key = "cons:pop:drained"
|
||||||
|
|
||||||
|
assert {3, conn} = Client.command(conn, ["SADD", key, "a", "b", "c"])
|
||||||
|
|
||||||
|
{taken, conn} = Client.command(conn, ["SPOP", key, "3"])
|
||||||
|
assert Enum.sort(taken) == ["a", "b", "c"]
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", key])
|
||||||
|
|
||||||
|
log = flushedLog()
|
||||||
|
refute log =~ "SPOP"
|
||||||
|
|
||||||
|
conn = replayFromScratch(conn)
|
||||||
|
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", key])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "popping from a missing set replays as a no-op", %{conn: conn} do
|
||||||
|
assert {nil, conn} = Client.command(conn, ["SPOP", "cons:pop:ghost"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "cons:pop:marker", "v"])
|
||||||
|
|
||||||
|
flushedLog()
|
||||||
|
conn = replayFromScratch(conn)
|
||||||
|
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "cons:pop:ghost"])
|
||||||
|
assert {"v", _conn} = Client.command(conn, ["GET", "cons:pop:marker"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "OBJECT IDLETIME must report the tracked idle time" do
|
||||||
|
test "a key left alone for five seconds reports five", %{conn: conn} do
|
||||||
|
assert :ok = Config.set("maxmemory-policy", "allkeys-lru")
|
||||||
|
|
||||||
|
key = "cons:idle:tracked"
|
||||||
|
assert :ok = Store.set(key, "v", [])
|
||||||
|
Tracker.touch(Tracker, key, System.system_time(:millisecond) - 5_000)
|
||||||
|
|
||||||
|
assert {5, _conn} = Client.command(conn, ["OBJECT", "IDLETIME", key])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a key without access metadata reports zero rather than an error", %{conn: conn} do
|
||||||
|
key = "cons:idle:untracked"
|
||||||
|
assert :ok = Store.set(key, "v", [])
|
||||||
|
|
||||||
|
assert {0, _conn} = Client.command(conn, ["OBJECT", "IDLETIME", key])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a missing key is still refused", %{conn: conn} do
|
||||||
|
assert {{:error, "ERR no such key"}, _conn} =
|
||||||
|
Client.command(conn, ["OBJECT", "IDLETIME", "cons:idle:ghost"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp duringSuspendedShard(shardKey, work) do
|
||||||
|
shard = shardPid(shardKey)
|
||||||
|
parent = self()
|
||||||
|
|
||||||
|
:sys.suspend(shard)
|
||||||
|
|
||||||
|
worker =
|
||||||
|
Task.async(fn ->
|
||||||
|
send(parent, :entered)
|
||||||
|
work.()
|
||||||
|
end)
|
||||||
|
|
||||||
|
queued =
|
||||||
|
try do
|
||||||
|
assert_receive :entered, 100
|
||||||
|
waiting = awaitQueuedCall(shard, worker.pid)
|
||||||
|
Process.sleep(@holdMs)
|
||||||
|
waiting
|
||||||
|
after
|
||||||
|
:sys.resume(shard)
|
||||||
|
end
|
||||||
|
|
||||||
|
assert queued, "the worker never reached the suspended shard, the expiry window was not open"
|
||||||
|
|
||||||
|
Task.await(worker)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp awaitQueuedCall(shard, workerPid) do
|
||||||
|
Enum.any?(1..500, fn _attempt ->
|
||||||
|
if callPending?(shard, workerPid) do
|
||||||
|
true
|
||||||
|
else
|
||||||
|
Process.sleep(1)
|
||||||
|
false
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp callPending?(shard, workerPid) do
|
||||||
|
{:messages, pending} = Process.info(shard, :messages)
|
||||||
|
Enum.any?(pending, &match?({:"$gen_call", {^workerPid, _ref}, _request}, &1))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp flushedLog do
|
||||||
|
Aof.flush()
|
||||||
|
{:ok, log} = File.read(Application.get_env(:excache, :aof_path))
|
||||||
|
log
|
||||||
|
end
|
||||||
|
|
||||||
|
defp replayFromScratch(conn) do
|
||||||
|
Store.flushdb()
|
||||||
|
Aof.replay(Application.get_env(:excache, :aof_path))
|
||||||
|
conn
|
||||||
|
end
|
||||||
|
|
||||||
|
defp shardIndex(key), do: rem(:erlang.phash2(key), Store.shard_count())
|
||||||
|
|
||||||
|
defp shardPid(key), do: Process.whereis(Store.shard_name(shardIndex(key)))
|
||||||
|
|
||||||
|
defp keyInOtherShard(prefix, reference) do
|
||||||
|
Enum.find_value(1..1024, fn suffix ->
|
||||||
|
candidate = prefix <> Integer.to_string(suffix)
|
||||||
|
if shardIndex(candidate) != shardIndex(reference), do: candidate
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,347 @@
|
|||||||
|
defmodule Excache.StoreOpsRegressionTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
describe "SMOVE must not drop the member when the destination has the wrong type" do
|
||||||
|
test "the member stays in the source and the caller is told about the type clash" do
|
||||||
|
source = "regr:smove:src"
|
||||||
|
dest = "regr:smove:dst-string"
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.sadd(source, ["alpha"])
|
||||||
|
assert :ok = Store.set(dest, "not-a-set", [])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.smove(source, dest, "alpha")
|
||||||
|
assert {:ok, ["alpha"]} = Store.smembers(source)
|
||||||
|
assert {:ok, "not-a-set"} = Store.get(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the destination type is checked before the source is touched" do
|
||||||
|
source = "regr:smove:absent-src"
|
||||||
|
dest = "regr:smove:dst-list"
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.rpush(dest, ["element"])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.smove(source, dest, "alpha")
|
||||||
|
assert {:ok, ["element"]} = Store.lrange(dest, 0, -1)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a wrong type on the source is reported and the destination is left alone" do
|
||||||
|
source = "regr:smove:src-string"
|
||||||
|
dest = "regr:smove:dst-set"
|
||||||
|
|
||||||
|
assert :ok = Store.set(source, "not-a-set", [])
|
||||||
|
assert {:ok, 1} = Store.sadd(dest, ["beta"])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.smove(source, dest, "alpha")
|
||||||
|
assert {:ok, ["beta"]} = Store.smembers(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a healthy move across two shards still transfers the member exactly once" do
|
||||||
|
source = "regr:smove:move-src"
|
||||||
|
dest = keyInOtherShard("regr:smove:move-dst-", source)
|
||||||
|
|
||||||
|
assert {:ok, 2} = Store.sadd(source, ["alpha", "gamma"])
|
||||||
|
assert {:ok, 1} = Store.sadd(dest, ["beta"])
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.smove(source, dest, "alpha")
|
||||||
|
assert {:ok, ["gamma"]} = Store.smembers(source)
|
||||||
|
assert {:ok, members} = Store.smembers(dest)
|
||||||
|
assert Enum.sort(members) == ["alpha", "beta"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "moving a member the source does not hold reports zero and adds nothing" do
|
||||||
|
source = "regr:smove:miss-src"
|
||||||
|
dest = "regr:smove:miss-dst"
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.sadd(source, ["alpha"])
|
||||||
|
assert {:ok, 1} = Store.sadd(dest, ["beta"])
|
||||||
|
|
||||||
|
assert {:ok, 0} = Store.smove(source, dest, "nothing-like-this")
|
||||||
|
assert {:ok, ["alpha"]} = Store.smembers(source)
|
||||||
|
assert {:ok, ["beta"]} = Store.smembers(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SMOVE over RESP reports WRONGTYPE and keeps the member", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["SADD", "regr:smove:resp-src", "alpha"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "regr:smove:resp-dst", "s"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, [
|
||||||
|
"SMOVE",
|
||||||
|
"regr:smove:resp-src",
|
||||||
|
"regr:smove:resp-dst",
|
||||||
|
"alpha"
|
||||||
|
])
|
||||||
|
|
||||||
|
assert {["alpha"], _conn} = Client.command(conn, ["SMEMBERS", "regr:smove:resp-src"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "RPOPLPUSH must not drop the element when the destination has the wrong type" do
|
||||||
|
test "the element stays at the tail of the source and the caller sees the error" do
|
||||||
|
source = "regr:rpoplpush:src"
|
||||||
|
dest = "regr:rpoplpush:dst-string"
|
||||||
|
|
||||||
|
assert {:ok, 2} = Store.rpush(source, ["head", "tail"])
|
||||||
|
assert :ok = Store.set(dest, "not-a-list", [])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.rpoplpush(source, dest)
|
||||||
|
assert {:ok, ["head", "tail"]} = Store.lrange(source, 0, -1)
|
||||||
|
assert {:ok, "not-a-list"} = Store.get(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the destination type is checked before the source is popped" do
|
||||||
|
source = "regr:rpoplpush:absent-src"
|
||||||
|
dest = "regr:rpoplpush:dst-set"
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.sadd(dest, ["beta"])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.rpoplpush(source, dest)
|
||||||
|
assert {:ok, ["beta"]} = Store.smembers(dest)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a healthy rotation across two shards moves the tail to the head" do
|
||||||
|
source = "regr:rpoplpush:rot-src"
|
||||||
|
dest = keyInOtherShard("regr:rpoplpush:rot-dst-", source)
|
||||||
|
|
||||||
|
assert {:ok, 2} = Store.rpush(source, ["head", "tail"])
|
||||||
|
assert {:ok, 1} = Store.rpush(dest, ["existing"])
|
||||||
|
|
||||||
|
assert {:ok, "tail"} = Store.rpoplpush(source, dest)
|
||||||
|
assert {:ok, ["head"]} = Store.lrange(source, 0, -1)
|
||||||
|
assert {:ok, ["tail", "existing"]} = Store.lrange(dest, 0, -1)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an empty source yields nil and never creates the destination" do
|
||||||
|
source = "regr:rpoplpush:empty-src"
|
||||||
|
dest = "regr:rpoplpush:empty-dst"
|
||||||
|
|
||||||
|
assert {:ok, nil} = Store.rpoplpush(source, dest)
|
||||||
|
assert 0 == Store.exists([dest])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "RPOPLPUSH over RESP reports WRONGTYPE and keeps the element", %{conn: conn} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "regr:rpoplpush:resp-src", "tail"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["SET", "regr:rpoplpush:resp-dst", "s"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, [
|
||||||
|
"RPOPLPUSH",
|
||||||
|
"regr:rpoplpush:resp-src",
|
||||||
|
"regr:rpoplpush:resp-dst"
|
||||||
|
])
|
||||||
|
|
||||||
|
assert {["tail"], _conn} =
|
||||||
|
Client.command(conn, ["LRANGE", "regr:rpoplpush:resp-src", "0", "-1"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "LPOS with a nil count must not crash the shard" do
|
||||||
|
test "a nil count means the first match and the shard survives the call" do
|
||||||
|
key = "regr:lpos:nil-count"
|
||||||
|
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
|
||||||
|
shardBefore = shardPid(key)
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.lpos(key, "b", 1, nil, 0)
|
||||||
|
|
||||||
|
assert shardPid(key) == shardBefore
|
||||||
|
assert Process.alive?(shardBefore)
|
||||||
|
assert {:ok, ["a", "b", "c", "b"]} = Store.lrange(key, 0, -1)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a nil count must not take the shard down and wipe its other keys" do
|
||||||
|
key = "regr:lpos:blast-radius"
|
||||||
|
neighbour = keyInSameShard("regr:lpos:neighbour-", key)
|
||||||
|
|
||||||
|
assert :ok = Store.set(neighbour, "survivor", [])
|
||||||
|
assert {:ok, 1} = Store.rpush(key, ["only"])
|
||||||
|
shardBefore = shardPid(key)
|
||||||
|
|
||||||
|
assert {:ok, 0} = Store.lpos(key, "only", 1, nil, 0)
|
||||||
|
|
||||||
|
assert shardPid(key) == shardBefore
|
||||||
|
assert {:ok, "survivor"} = Store.get(neighbour)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a nil count with no match yields nil rather than an empty list" do
|
||||||
|
key = "regr:lpos:no-match"
|
||||||
|
assert {:ok, 3} = Store.rpush(key, ["a", "b", "c"])
|
||||||
|
|
||||||
|
assert {:ok, nil} = Store.lpos(key, "zzz", 1, nil, 0)
|
||||||
|
assert {:ok, nil} = Store.lpos("regr:lpos:missing-key", "a", 1, nil, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a nil count honours a positive rank" do
|
||||||
|
key = "regr:lpos:rank-positive"
|
||||||
|
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.lpos(key, "b", 1, nil, 0)
|
||||||
|
assert {:ok, 3} = Store.lpos(key, "b", 2, nil, 0)
|
||||||
|
assert {:ok, nil} = Store.lpos(key, "b", 3, nil, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a nil count honours a negative rank and searches from the tail" do
|
||||||
|
key = "regr:lpos:rank-negative"
|
||||||
|
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
|
||||||
|
|
||||||
|
assert {:ok, 3} = Store.lpos(key, "b", -1, nil, 0)
|
||||||
|
assert {:ok, 1} = Store.lpos(key, "b", -2, nil, 0)
|
||||||
|
assert {:ok, nil} = Store.lpos(key, "b", -3, nil, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a nil count honours maxlen" do
|
||||||
|
key = "regr:lpos:maxlen"
|
||||||
|
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
|
||||||
|
|
||||||
|
assert {:ok, 1} = Store.lpos(key, "b", 1, nil, 2)
|
||||||
|
assert {:ok, nil} = Store.lpos(key, "b", 1, nil, 1)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a zero count still means every match, in both rank directions" do
|
||||||
|
key = "regr:lpos:count-zero"
|
||||||
|
assert {:ok, 4} = Store.rpush(key, ["a", "b", "c", "b"])
|
||||||
|
|
||||||
|
assert {:ok, [1, 3]} = Store.lpos(key, "b", 1, 0, 0)
|
||||||
|
assert {:ok, [3, 1]} = Store.lpos(key, "b", -1, 0, 0)
|
||||||
|
assert {:ok, [1]} = Store.lpos(key, "b", 1, 1, 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a nil count on a wrong type key reports WRONGTYPE instead of crashing" do
|
||||||
|
key = "regr:lpos:wrongtype"
|
||||||
|
assert :ok = Store.set(key, "not-a-list", [])
|
||||||
|
shardBefore = shardPid(key)
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.lpos(key, "a", 1, nil, 0)
|
||||||
|
assert shardPid(key) == shardBefore
|
||||||
|
end
|
||||||
|
|
||||||
|
test "LPOS over RESP is unaffected by the fix", %{conn: conn} do
|
||||||
|
assert {4, conn} = Client.command(conn, ["RPUSH", "regr:lpos:resp", "a", "b", "c", "b"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "b"])
|
||||||
|
assert {nil, conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "zzz"])
|
||||||
|
assert {3, conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "b", "RANK", "-1"])
|
||||||
|
assert {[1, 3], _conn} = Client.command(conn, ["LPOS", "regr:lpos:resp", "b", "COUNT", "0"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "HMGET and SMISMEMBER must surface WRONGTYPE" do
|
||||||
|
test "HMGET on a key of the wrong type reports the error instead of nils" do
|
||||||
|
key = "regr:hmget:wrongtype"
|
||||||
|
assert :ok = Store.set(key, "not-a-hash", [])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.hmget(key, ["field-a", "field-b"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "HMGET keeps returning a bare list for hashes and missing keys" do
|
||||||
|
key = "regr:hmget:hash"
|
||||||
|
assert {:ok, 1} = Store.hset(key, [{"field-a", "value-a"}])
|
||||||
|
|
||||||
|
assert ["value-a", nil] = Store.hmget(key, ["field-a", "field-b"])
|
||||||
|
assert [nil, nil] = Store.hmget("regr:hmget:missing", ["field-a", "field-b"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SMISMEMBER on a key of the wrong type reports the error instead of zeros" do
|
||||||
|
key = "regr:smismember:wrongtype"
|
||||||
|
assert :ok = Store.set(key, "not-a-set", [])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.smismember(key, ["alpha", "beta"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SMISMEMBER keeps returning a bare list for sets and missing keys" do
|
||||||
|
key = "regr:smismember:set"
|
||||||
|
assert {:ok, 1} = Store.sadd(key, ["alpha"])
|
||||||
|
|
||||||
|
assert [1, 0] = Store.smismember(key, ["alpha", "beta"])
|
||||||
|
assert [0, 0] = Store.smismember("regr:smismember:missing", ["alpha", "beta"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "HMGET and SMISMEMBER over RESP still report WRONGTYPE", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "regr:mixed:string", "v"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, conn} =
|
||||||
|
Client.command(conn, ["HMGET", "regr:mixed:string", "field-a"])
|
||||||
|
|
||||||
|
assert {{:error, "WRONGTYPE" <> _}, _conn} =
|
||||||
|
Client.command(conn, ["SMISMEMBER", "regr:mixed:string", "alpha"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "SETRANGE with an empty value must not create a key" do
|
||||||
|
test "an empty value on a missing key returns zero and creates nothing" do
|
||||||
|
key = "regr:setrange:missing"
|
||||||
|
|
||||||
|
assert {:ok, 0} = Store.setrange(key, 5, "")
|
||||||
|
assert 0 == Store.exists([key])
|
||||||
|
assert "none" == Store.type(key)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an empty value on an existing string returns its length and changes nothing" do
|
||||||
|
key = "regr:setrange:existing"
|
||||||
|
assert :ok = Store.set(key, "hello", [])
|
||||||
|
|
||||||
|
assert {:ok, 5} = Store.setrange(key, 100, "")
|
||||||
|
assert {:ok, "hello"} = Store.get(key)
|
||||||
|
assert 1 == Store.exists([key])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an empty value on a key of the wrong type still reports WRONGTYPE" do
|
||||||
|
key = "regr:setrange:wrongtype"
|
||||||
|
assert {:ok, 1} = Store.rpush(key, ["element"])
|
||||||
|
|
||||||
|
assert {:error, :wrongtype} = Store.setrange(key, 0, "")
|
||||||
|
assert {:ok, ["element"]} = Store.lrange(key, 0, -1)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an empty value leaves an existing expiry in place" do
|
||||||
|
key = "regr:setrange:ttl"
|
||||||
|
assert :ok = Store.set(key, "hello", [])
|
||||||
|
assert 1 == Store.expire(key, 60_000)
|
||||||
|
|
||||||
|
assert {:ok, 5} = Store.setrange(key, 0, "")
|
||||||
|
assert Store.ttl(key, :seconds) > 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a non empty value still pads and overwrites as before" do
|
||||||
|
key = "regr:setrange:write"
|
||||||
|
|
||||||
|
assert {:ok, 8} = Store.setrange(key, 5, "abc")
|
||||||
|
assert {:ok, <<0, 0, 0, 0, 0, "abc">>} = Store.get(key)
|
||||||
|
assert {:ok, 8} = Store.setrange(key, 0, "xy")
|
||||||
|
assert {:ok, <<"xy", 0, 0, 0, "abc">>} = Store.get(key)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "SETRANGE with an empty value over RESP returns the length and creates nothing",
|
||||||
|
%{conn: conn} do
|
||||||
|
assert {0, conn} = Client.command(conn, ["SETRANGE", "regr:setrange:resp", "5", ""])
|
||||||
|
assert {0, conn} = Client.command(conn, ["EXISTS", "regr:setrange:resp"])
|
||||||
|
|
||||||
|
assert {{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["SET", "regr:setrange:resp", "hello"])
|
||||||
|
|
||||||
|
assert {5, _conn} = Client.command(conn, ["SETRANGE", "regr:setrange:resp", "100", ""])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp shardIndex(key), do: rem(:erlang.phash2(key), Store.shard_count())
|
||||||
|
|
||||||
|
defp shardPid(key), do: Process.whereis(Store.shard_name(shardIndex(key)))
|
||||||
|
|
||||||
|
defp keyInOtherShard(prefix, reference) do
|
||||||
|
findKey(prefix, fn candidate -> shardIndex(candidate) != shardIndex(reference) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp keyInSameShard(prefix, reference) do
|
||||||
|
findKey(prefix, fn candidate -> shardIndex(candidate) == shardIndex(reference) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp findKey(prefix, accept) do
|
||||||
|
Enum.find_value(1..1024, fn suffix ->
|
||||||
|
candidate = prefix <> Integer.to_string(suffix)
|
||||||
|
if accept.(candidate), do: candidate
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,265 @@
|
|||||||
|
defmodule Excache.WritePathTest do
|
||||||
|
use Excache.Test.ServerCase, async: false
|
||||||
|
|
||||||
|
alias Excache.Aof
|
||||||
|
alias Excache.Blocking
|
||||||
|
alias Excache.Command
|
||||||
|
alias Excache.Config
|
||||||
|
alias Excache.Eviction.Runner
|
||||||
|
alias Excache.Eviction.Tracker
|
||||||
|
alias Excache.Store
|
||||||
|
|
||||||
|
@payload String.duplicate("x", 1000)
|
||||||
|
@oom {:error, "OOM command not allowed when used memory > 'maxmemory'"}
|
||||||
|
|
||||||
|
describe "renaming a key onto itself keeps it" do
|
||||||
|
test "the store answers OK and the value survives" do
|
||||||
|
assert :ok = Store.set("self:string", "payload", [])
|
||||||
|
assert {:ok, "OK"} = Store.rename("self:string", "self:string")
|
||||||
|
assert {:ok, "payload"} = Store.get("self:string")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a missing key renamed onto itself is still refused" do
|
||||||
|
assert {:error, :nokey} = Store.rename("self:ghost", "self:ghost")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the remaining ttl is untouched" do
|
||||||
|
assert :ok = Store.set("self:ttl", "payload", px: 60_000)
|
||||||
|
assert {:ok, "OK"} = Store.rename("self:ttl", "self:ttl")
|
||||||
|
|
||||||
|
remaining = Store.ttl("self:ttl", :seconds)
|
||||||
|
assert remaining > 0 and remaining <= 60
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a list survives the round trip over the wire", %{conn: conn} do
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "self:list", "a", "b"])
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["RENAME", "self:list", "self:list"])
|
||||||
|
assert {["a", "b"], conn} = Client.command(conn, ["LRANGE", "self:list", "0", "-1"])
|
||||||
|
|
||||||
|
conn = replayFromScratch(conn)
|
||||||
|
|
||||||
|
assert {["a", "b"], _conn} = Client.command(conn, ["LRANGE", "self:list", "0", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "RENAMENX onto itself answers zero and keeps the key", %{conn: conn} do
|
||||||
|
assert {{:simple, "OK"}, conn} = Client.command(conn, ["SET", "self:nx", "v"])
|
||||||
|
assert {0, conn} = Client.command(conn, ["RENAMENX", "self:nx", "self:nx"])
|
||||||
|
assert {"v", _conn} = Client.command(conn, ["GET", "self:nx"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "blocking moves obey the out of memory gate" do
|
||||||
|
setup do
|
||||||
|
Application.put_env(:excache, :eviction_recheck, 0)
|
||||||
|
Config.reset()
|
||||||
|
Tracker.clear(Tracker)
|
||||||
|
Runner.resetStats()
|
||||||
|
|
||||||
|
on_exit(fn ->
|
||||||
|
Config.reset()
|
||||||
|
Application.delete_env(:excache, :eviction_recheck)
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BRPOPLPUSH is refused once maxmemory is crossed", %{conn: conn} do
|
||||||
|
conn = fillOverLimit(conn, "oom:brpoplpush")
|
||||||
|
|
||||||
|
assert {@oom, conn} =
|
||||||
|
Client.command(conn, ["BRPOPLPUSH", "oom:brpoplpush", "oom:moved", "0"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LLEN", "oom:brpoplpush"])
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "oom:moved"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "BLMOVE is refused once maxmemory is crossed", %{conn: conn} do
|
||||||
|
conn = fillOverLimit(conn, "oom:blmove")
|
||||||
|
|
||||||
|
assert {@oom, conn} =
|
||||||
|
Client.command(conn, ["BLMOVE", "oom:blmove", "oom:moved", "LEFT", "RIGHT", "0"])
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LLEN", "oom:blmove"])
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "oom:moved"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "lifting the limit lets the same move through", %{conn: conn} do
|
||||||
|
conn = fillOverLimit(conn, "oom:lifted")
|
||||||
|
|
||||||
|
assert {@oom, conn} =
|
||||||
|
Client.command(conn, ["BLMOVE", "oom:lifted", "oom:dst", "LEFT", "RIGHT", "0"])
|
||||||
|
|
||||||
|
{{:simple, "OK"}, conn} = Client.command(conn, ["CONFIG", "SET", "maxmemory", "0"])
|
||||||
|
|
||||||
|
assert {"value", conn} =
|
||||||
|
Client.command(conn, ["BLMOVE", "oom:lifted", "oom:dst", "LEFT", "RIGHT", "0"])
|
||||||
|
|
||||||
|
assert {["value"], _conn} = Client.command(conn, ["LRANGE", "oom:dst", "0", "-1"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "blocking writes reach the shared log funnel" do
|
||||||
|
test "a blocked BLMOVE announces the push it performed", %{conn: conn, port: port} do
|
||||||
|
subscriber = subscribeTo(port, "__keyevent@0__:rpush")
|
||||||
|
|
||||||
|
waiter = blockingTask(port, ["BLMOVE", "funnel:src", "funnel:dst", "LEFT", "RIGHT", "0"])
|
||||||
|
awaitWaiters("funnel:src", 1)
|
||||||
|
|
||||||
|
assert {1, _conn} = Client.command(conn, ["LPUSH", "funnel:src", "item"])
|
||||||
|
assert Task.await(waiter) == "item"
|
||||||
|
|
||||||
|
assert {["message", "__keyevent@0__:rpush", "funnel:dst"], _subscriber} =
|
||||||
|
Client.read(subscriber)
|
||||||
|
|
||||||
|
Client.close(subscriber)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "an immediately served BLMOVE announces the push as well", %{conn: conn, port: port} do
|
||||||
|
assert {1, conn} = Client.command(conn, ["RPUSH", "inline:src", "item"])
|
||||||
|
|
||||||
|
subscriber = subscribeTo(port, "__keyevent@0__:rpush")
|
||||||
|
|
||||||
|
assert {"item", _conn} =
|
||||||
|
Client.command(conn, ["BLMOVE", "inline:src", "inline:dst", "LEFT", "RIGHT", "0"])
|
||||||
|
|
||||||
|
assert {["message", "__keyevent@0__:rpush", "inline:dst"], _subscriber} =
|
||||||
|
Client.read(subscriber)
|
||||||
|
|
||||||
|
Client.close(subscriber)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a blocked BLMOVE replays into the very same lists", %{conn: conn, port: port} do
|
||||||
|
waiter = blockingTask(port, ["BLMOVE", "replay:src", "replay:dst", "LEFT", "RIGHT", "0"])
|
||||||
|
awaitWaiters("replay:src", 1)
|
||||||
|
|
||||||
|
assert {2, conn} = Client.command(conn, ["RPUSH", "replay:src", "moved", "stays"])
|
||||||
|
assert Task.await(waiter) == "moved"
|
||||||
|
|
||||||
|
{source, conn} = Client.command(conn, ["LRANGE", "replay:src", "0", "-1"])
|
||||||
|
{destination, conn} = Client.command(conn, ["LRANGE", "replay:dst", "0", "-1"])
|
||||||
|
|
||||||
|
log = flushedLog()
|
||||||
|
refute log =~ "BLMOVE"
|
||||||
|
|
||||||
|
conn = replayFromScratch(conn)
|
||||||
|
|
||||||
|
assert {^source, conn} = Client.command(conn, ["LRANGE", "replay:src", "0", "-1"])
|
||||||
|
assert {^destination, _conn} = Client.command(conn, ["LRANGE", "replay:dst", "0", "-1"])
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the pop of a woken BLPOP is logged after the push that woke it", %{
|
||||||
|
conn: conn,
|
||||||
|
port: port
|
||||||
|
} do
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "order:queue", "0"])
|
||||||
|
awaitWaiters("order:queue", 1)
|
||||||
|
|
||||||
|
assert {1, conn} = Client.command(conn, ["LPUSH", "order:queue", "payload"])
|
||||||
|
assert Task.await(waiter) == ["order:queue", "payload"]
|
||||||
|
|
||||||
|
log = flushedLog()
|
||||||
|
|
||||||
|
assert offsetOf(log, "LPUSH") < offsetOf(log, "LPOP")
|
||||||
|
|
||||||
|
conn = replayFromScratch(conn)
|
||||||
|
|
||||||
|
assert {0, _conn} = Client.command(conn, ["EXISTS", "order:queue"])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "the wake signal belongs to the command layer" do
|
||||||
|
test "a push dispatched straight through the command router wakes a waiter", %{port: port} do
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "hook:queue", "2"])
|
||||||
|
awaitWaiters("hook:queue", 1)
|
||||||
|
|
||||||
|
Command.execute(["LPUSH", "hook:queue", "direct"], commandState())
|
||||||
|
|
||||||
|
assert Task.await(waiter) == ["hook:queue", "direct"]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a rename dispatched straight through the command router wakes a waiter", %{port: port} do
|
||||||
|
assert {:ok, 1} = Store.rpush("hook:source", ["carried"])
|
||||||
|
|
||||||
|
waiter = blockingTask(port, ["BLPOP", "hook:target", "2"])
|
||||||
|
awaitWaiters("hook:target", 1)
|
||||||
|
|
||||||
|
Command.execute(["RENAME", "hook:source", "hook:target"], commandState())
|
||||||
|
|
||||||
|
assert Task.await(waiter) == ["hook:target", "carried"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp commandState do
|
||||||
|
Map.put(Excache.Command.Transaction.initial_state(), :subscriptions, MapSet.new())
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fillOverLimit(conn, listKey) do
|
||||||
|
conn =
|
||||||
|
Enum.reduce(1..30, conn, fn index, acc ->
|
||||||
|
{{:simple, "OK"}, acc} = Client.command(acc, ["SET", "oom:key:#{index}", @payload])
|
||||||
|
acc
|
||||||
|
end)
|
||||||
|
|
||||||
|
{1, conn} = Client.command(conn, ["RPUSH", listKey, "value"])
|
||||||
|
{flat, conn} = Client.command(conn, ["MEMORY", "STATS"])
|
||||||
|
|
||||||
|
allocated =
|
||||||
|
flat
|
||||||
|
|> Enum.chunk_every(2)
|
||||||
|
|> Map.new(fn [name, value] -> {name, value} end)
|
||||||
|
|> Map.fetch!("total.allocated")
|
||||||
|
|
||||||
|
{{:simple, "OK"}, conn} =
|
||||||
|
Client.command(conn, ["CONFIG", "SET", "maxmemory", Integer.to_string(div(allocated, 2))])
|
||||||
|
|
||||||
|
conn
|
||||||
|
end
|
||||||
|
|
||||||
|
defp subscribeTo(port, channel) do
|
||||||
|
subscriber = Client.connect(port)
|
||||||
|
{["subscribe", ^channel, 1], subscriber} = Client.command(subscriber, ["SUBSCRIBE", channel])
|
||||||
|
subscriber
|
||||||
|
end
|
||||||
|
|
||||||
|
defp blockingTask(port, args) do
|
||||||
|
Task.async(fn ->
|
||||||
|
client = Client.connect(port)
|
||||||
|
{reply, _client} = Client.command(client, args)
|
||||||
|
Client.close(client)
|
||||||
|
reply
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp awaitWaiters(key, count), do: awaitUntil(fn -> Blocking.waitingCount(key) == count end)
|
||||||
|
|
||||||
|
defp awaitUntil(check, attempts \\ 400) do
|
||||||
|
cond do
|
||||||
|
check.() ->
|
||||||
|
:ok
|
||||||
|
|
||||||
|
attempts == 0 ->
|
||||||
|
flunk("the expected registry state was never reached")
|
||||||
|
|
||||||
|
true ->
|
||||||
|
Process.sleep(5)
|
||||||
|
awaitUntil(check, attempts - 1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp offsetOf(log, needle) do
|
||||||
|
{offset, _length} = :binary.match(log, needle)
|
||||||
|
offset
|
||||||
|
end
|
||||||
|
|
||||||
|
defp flushedLog do
|
||||||
|
Aof.flush()
|
||||||
|
{:ok, log} = File.read(Application.get_env(:excache, :aof_path))
|
||||||
|
log
|
||||||
|
end
|
||||||
|
|
||||||
|
defp replayFromScratch(conn) do
|
||||||
|
Store.flushdb()
|
||||||
|
Aof.replay(Application.get_env(:excache, :aof_path))
|
||||||
|
conn
|
||||||
|
end
|
||||||
|
end
|
||||||
Reference in new issue
Block a user