From 346d4f003faa4d41195a22d579ada27cba31a3b2 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Mon, 9 Nov 2020 17:11:39 -0600 Subject: [PATCH 01/28] .gitignore for .elixir_ls --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 45ea1f8..b21d4b0 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ erl_crash.dump *.ez .bundle + +.elixir_ls \ No newline at end of file From 0a40c9324ea04495e4525973ec13236c69a0a548 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Mon, 9 Nov 2020 23:50:05 -0600 Subject: [PATCH 02/28] Starting over... again --- lib/cream/connection.ex | 138 ++++++++++++++++++++++++++++++ lib/cream/packet.ex | 168 +++++++++++++++++++++++++++++++++++++ mix.exs | 9 +- mix.lock | 4 +- test/cream_test.exs | 133 ----------------------------- test/low_level_test.exs | 180 ++++++++++++++++++++++++++++++++++++++++ test/test_helper.exs | 1 - 7 files changed, 492 insertions(+), 141 deletions(-) create mode 100644 lib/cream/connection.ex create mode 100644 lib/cream/packet.ex delete mode 100644 test/cream_test.exs create mode 100644 test/low_level_test.exs diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex new file mode 100644 index 0000000..3875f06 --- /dev/null +++ b/lib/cream/connection.ex @@ -0,0 +1,138 @@ +defmodule Cream.Connection do + use Connection + require Logger + alias Cream.Packet + + @defaults [ + server: "localhost:11211" + ] + + def start_link(options \\ []) do + options = Keyword.merge(@defaults, options) + Connection.start_link(__MODULE__, options) + end + + def send_packets(conn, packets) do + Connection.call(conn, {:send_packets, packets}) + end + + def recv_packets(conn, count) do + Connection.call(conn, {:recv_packets, count}) + end + + def get(conn, key, options \\ []) do + args = Keyword.merge(options, key: key) + with :ok <- send_packets(conn, [Packet.new(:get, args)]), + {:ok, [packet]} <- recv_packets(conn, 1) + do + case packet.status do + :ok -> packet.value + :not_found -> nil + end + else + error -> error + end + end + + def set(conn, key, value, options \\ []) do + args = Keyword.merge(options, key: key, value: value) + with :ok <- send_packets(conn, [Packet.new(:set, args)]), + {:ok, [packet]} <- recv_packets(conn, 1) + do + packet.status + else + error -> error + end + end + + def init(options) do + state = %{ + options: options, + socket: nil, + } + + {:connect, :init, state} + end + + def connect(_context, state) do + server = state.options[:server] + url = "tcp://#{server}" + + case Socket.connect(url) do + {:ok, socket} -> {:ok, %{state | socket: socket}} + {:error, reason} -> + Logger.warn("#{server} #{reason}") + {:backoff, 1000, state} + end + end + + def handle_call({:send_packets, packets}, _from, state) do + {:reply, do_send_packets(state.socket, packets), state} + end + + def handle_call({:recv_packets, count}, _from, state) do + {:reply, do_recv_packets(state.socket, count), state} + end + + defp do_send_packets(socket, packets) do + Enum.reduce_while(packets, :ok, fn packet, _result -> + case Socket.Stream.send(socket, Packet.serialize(packet)) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) + end + + defp do_recv_packets(socket, :noop) do + Stream.repeatedly(fn -> do_recv_packet(socket) end) + |> Enum.reduce_while([], fn + {:ok, packet}, packets -> if packet.opcode == :noop do + {:halt, [packet | packets]} + else + {:cont, [packet | packets]} + end + error, _packets -> {:halt, error} + end) + |> case do + {:error, reason} -> {:error, reason} + packets -> {:ok, Enum.reverse(packets)} + end + end + + defp do_recv_packets(socket, count) do + Enum.reduce_while(1..count, [], fn _i, packets -> + case do_recv_packet(socket) do + {:ok, packet} -> {:cont, [packet | packets]} + error -> {:halt, error} + end + end) + |> case do + {:error, reason} -> {:error, reason} + packets -> {:ok, Enum.reverse(packets)} + end + end + + defp do_recv_packet(socket) do + with {:ok, data} <- do_recv_header(socket), + packet = Packet.deserialize_header(data), + {:ok, data} <- do_recv_body(socket, packet) + do + {:ok, Packet.deserialize_body(packet, data)} + else + error -> error + end + end + + defp do_recv_header(socket) do + Socket.Stream.recv(socket, 24) + end + + defp do_recv_body(_socket, packet) when packet.total_body_length == 0 do + {:ok, ""} + end + + defp do_recv_body(socket, packet) when packet.total_body_length > 0 do + Socket.Stream.recv(socket, packet.total_body_length) + end + +end diff --git a/lib/cream/packet.ex b/lib/cream/packet.ex new file mode 100644 index 0000000..6b596f6 --- /dev/null +++ b/lib/cream/packet.ex @@ -0,0 +1,168 @@ +defmodule Cream.Packet do + + @request_magic 0x80 + @response_magic 0x81 + + defstruct [ + magic: @request_magic, + opcode: nil, + key_length: 0x0000, + extra_length: 0x00, + data_type: 0x00, + vbucket_id: 0x0000, + status: 0x0000, + total_body_length: 0x00000000, + opaque: 0x00000000, + cas: 0x0000000000000000, + extra: "", + key: "", + value: "" + ] + + @atom_to_opcode %{ + get: 0x00, + set: 0x01, + flush: 0x08, + getq: 0x09, + noop: 0x0a, + getk: 0x0c, + getkq: 0x0d, + setq: 0x11, + } + + @atom_to_status %{ + ok: 0x0000, + not_found: 0x0001, + exists: 0x0002, + too_large: 0x0003, + invalid_args: 0x0004, + not_stored: 0x0005, + non_numeric: 0x0006, + vbucket_error: 0x0007, + auth_error: 0x0008, + auth_cont: 0x0009, + unknown_cmd: 0x0081, + oom: 0x0082, + not_supported: 0x0083, + internal_error: 0x0084, + busy: 0x0085, + temp_failure: 0x0086 + } + + @opcode_to_atom Enum.map(@atom_to_opcode, fn {k, v} -> {v, k} end) |> Map.new() + @status_to_atom Enum.map(@atom_to_status, fn {k, v} -> {v, k} end) |> Map.new() + + defmacrop bytes(n) do + bytes = n*8 + quote do: size(unquote(bytes)) + end + + def new(opcode, options \\ []) do + if not Map.has_key?(@atom_to_opcode, opcode) do + raise "unknown opcode #{opcode}" + end + + packet = %__MODULE__{ + magic: @request_magic, + opcode: opcode, + } + + packet = %{packet | + extra: serialize_extra(opcode, options), + key: options[:key] || "", + value: options[:value] || "" + } + + extra_length = IO.iodata_length(packet.extra) + key_length = byte_size(packet.key) + value_length = byte_size(packet.value) + + %{packet | + extra_length: extra_length, + key_length: key_length, + total_body_length: extra_length + key_length + value_length + } + end + + def serialize_extra(opcode, options) when opcode in [:set, :setq] do + flags = options[:flags] || 0 + expiration = options[:expiration] || 0 + + [ + <>, + <>, + ] + end + + def serialize_extra(_opcode, _options), do: [] + + def deserialize_extra(_opcode, ""), do: %{} + + def deserialize_extra(opcode, data) when opcode in [:get, :getq, :getk, :getkq] do + <> = data + %{flags: flags} + end + + def deserialize_extra(_opcode, _data), do: %{} + + def serialize(packet) when packet.magic == @request_magic do + opcode = @atom_to_opcode[packet.opcode] + + [ + <>, + <>, + <>, + <>, + <>, + <>, + <>, + <>, + <>, + packet.extra, + packet.key, + packet.value + ] + end + + def deserialize_header(data) do + << + @response_magic :: bytes(1), + opcode :: bytes(1), + key_length :: bytes(2), + extra_length :: bytes(1), + data_type :: bytes(1), + status :: bytes(2), + total_body_length :: bytes(4), + opaque :: bytes(4), + cas :: bytes(8) + >> = data + + %__MODULE__{ + magic: @response_magic, + opcode: @opcode_to_atom[opcode], + key_length: key_length, + extra_length: extra_length, + data_type: data_type, + status: @status_to_atom[status], + total_body_length: total_body_length, + opaque: opaque, + cas: cas + } + end + + def deserialize_body(packet, data) do + extra_length = packet.extra_length + key_length = packet.key_length + + << + extra :: binary-size(extra_length), + key :: binary-size(key_length), + value :: binary + >> = data + + extra = deserialize_extra(packet.opcode, extra) + + %{packet | extra: extra, key: key, value: value} + end + +end diff --git a/mix.exs b/mix.exs index fcdff1c..901b054 100644 --- a/mix.exs +++ b/mix.exs @@ -58,12 +58,9 @@ defmodule Cream.Mixfile do # Type "mix help deps" for more examples and options defp deps do [ - {:memcachex, ">= 0.4.0"}, - {:uuid, "~> 1.1"}, - {:poison, ">= 2.0.0"}, - {:poolboy, "~> 1.5"}, - {:ex_doc, "~> 0.0", only: :dev}, - {:instrumentation, ">= 0.1.0"} + {:telemetry, "~> 0.4.2"}, + {:connection, "~> 1.0"}, + {:socket, "~> 0.3.13"}, ] end end diff --git a/mix.lock b/mix.lock index c57f2cf..6427450 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,5 @@ %{ - "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], []}, + "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, "crc": {:hex, :crc, "0.5.2", "6db0c06f4bb2ae6a737a32b31fd40842774d4aae903b76e5f4dae44bd4b2742c", [:make, :mix], []}, "db_connection": {:hex, :db_connection, "1.1.0", "b2b88db6d7d12f99997b584d09fad98e560b817a20dab6a526830e339f54cdb3", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, "decimal": {:hex, :decimal, "1.3.1", "157b3cedb2bfcb5359372a7766dd7a41091ad34578296e951f58a946fcab49c6", [:mix], []}, @@ -13,5 +13,7 @@ "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []}, "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []}, "postgrex": {:hex, :postgrex, "0.12.1", "2f8b46cb3a44dcd42f42938abedbfffe7e103ba4ce810ccbeee8dcf27ca0fb06", [:mix], [{:connection, "~> 1.0", [hex: :connection, optional: false]}, {:db_connection, "~> 1.0-rc.4", [hex: :db_connection, optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, optional: false]}]}, + "socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"}, + "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, "uuid": {:hex, :uuid, "1.1.6", "4927232f244e69c6e255643014c2d639dad5b8313dc2a6976ee1c3724e6ca60d", [:mix], []}, } diff --git a/test/cream_test.exs b/test/cream_test.exs deleted file mode 100644 index d4f9f56..0000000 --- a/test/cream_test.exs +++ /dev/null @@ -1,133 +0,0 @@ -require IEx - -defmodule CreamTest do - use ExUnit.Case - - # import ExUnit.CaptureLog # Used to capture logging and assert against it. - - alias Test.Cluster - - setup do - Cluster.flush - :ok - end - - test "set and get" do - assert Cluster.get("name") == nil - Cluster.set({"name", "Callie"}) - assert Cluster.get("name") == "Callie" - end - - test "multi set / multi get" do - assert Cluster.get(["foo", "bar"]) == %{} - Cluster.set(%{"foo" => "oof", "bar" => "rab"}) - assert Cluster.get(["foo", "bar"]) == %{"foo" => "oof", "bar" => "rab"} - end - - test "multi get with missing key" do - assert Cluster.get(["foo", "bar"]) == %{} - Cluster.set(%{"foo" => "oof", "bar" => "rab"}) - assert Cluster.get(["foo", "bar", "baz"]) == %{"foo" => "oof", "bar" => "rab"} - end - - test "multi fetch with some missing keys" do - keys = ["foo", "bar", "baz"] - values = ["oof", "rab", "zab"] - expected = Enum.zip(keys, values) |> Enum.into(%{}) - - Cluster.set({"foo", "oof"}) - assert Cluster.get(keys) == %{"foo" => "oof"} - - results = Cluster.fetch keys, fn missing_keys -> - assert missing_keys == ["bar", "baz"] - Enum.map(missing_keys, &String.reverse/1) - end - - assert results == expected - assert Cluster.get(keys) == expected - end - - test "multi fetch with no missing keys" do - keys = ["foo", "bar", "baz"] - values = ["oof", "rab", "zab"] - expected = Enum.zip(keys, values) |> Enum.into(%{}) - - Cluster.set(expected) - assert Cluster.get(keys) == expected - - fetch_results = Cluster.fetch keys, fn _missing_keys -> - assert false - "this should not even be called" - end - - assert fetch_results == expected - assert Cluster.get(keys) == expected - end - - test "single fetch" do - assert Cluster.get("name") == nil - assert Cluster.fetch("name", fn -> "Callie" end) == "Callie" - assert Cluster.get("name") == "Callie" - end - - test "single fetch when key exists" do - Cluster.set {"name", "Callie"} - assert Cluster.get("name") == "Callie" - assert Cluster.fetch("name", fn -> "Not Callie" end) == "Callie" - assert Cluster.get("name") == "Callie" - end - - test "with conn" do - - keys = ~w(foo bar baz zip kip pik) - - results = keys - |> Cluster.with_conn(fn conn, keys -> - Enum.map(keys, &Memcache.get(conn, &1)) - end) - |> Map.values - |> List.flatten - - assert results == Enum.map(keys, fn _ -> {:error, "Key not found"} end) - end - - test "Dalli compatibility" do - {_, 0} = System.cmd("bundle", ~w(exec ruby test/support/populate.rb)) - - expected_hits = (0..19) - |> Enum.map(&{"cream_ruby_test_key_#{&1}", &1}) - |> Enum.into(%{}) - - hits = expected_hits - |> Map.keys - |> Cluster.get - - assert hits == expected_hits - end - - test "Dalli compatibility with JSON" do - {_, 0} = System.cmd("bundle", ~w(exec ruby test/support/populate.rb json)) - - expected = %{"one" => ["two", "three"]} - actual = Cluster.get("foo") - - assert expected == actual - end - - test "delete" do - Cluster.set({"foo", "bar"}) - assert Cluster.get("foo") == "bar" - Cluster.delete("foo") - assert Cluster.get("foo") == nil - end - - test "multi delete" do - Cluster.set([{"foo", "bar"}, {"one", "two"}]) - assert Cluster.get("foo") == "bar" - assert Cluster.get("one") == "two" - Cluster.delete(["foo", "one"]) - assert Cluster.get("foo") == nil - assert Cluster.get("one") == nil - end - -end diff --git a/test/low_level_test.exs b/test/low_level_test.exs new file mode 100644 index 0000000..560ba09 --- /dev/null +++ b/test/low_level_test.exs @@ -0,0 +1,180 @@ +defmodule LowLevelTest do + use ExUnit.Case + + alias Cream.{Connection, Packet} + + setup_all do + {:ok, conn} = Connection.start_link() + [conn: conn] + end + + test "noop", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:noop)]) + {:ok, [%{opcode: :noop}]} = Connection.recv_packets(conn, 1) + end + + test "flush", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:flush)]) + {:ok, [%{opcode: :flush}]} = Connection.recv_packets(conn, 1) + end + + describe "set, add, replace" do + + setup :flush + + test "set", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:set, key: "foo", value: "bar")]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :set + assert packet.status == :ok + end + + test "setq", %{conn: conn} do + :ok = Connection.send_packets(conn, [ + Packet.new(:setq, key: "foo", value: "bar"), + Packet.new(:set, key: "bar", value: "foo"), + ]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :set + assert packet.status == :ok + end + + end + + describe "retrieving (misses)" do + + setup :flush + + test "get", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:get, key: "foo")]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :get + assert packet.status == :not_found + assert packet.key == "" + assert packet.value == "Not found" + end + + test "getk", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:getk, key: "foo")]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :getk + assert packet.status == :not_found + assert packet.key == "foo" + assert packet.value == "" + end + + test "getq", %{conn: conn} do + :ok = Connection.send_packets(conn, [ + Packet.new(:getq, key: "foo"), + Packet.new(:getk, key: "bar") + ]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :getk + assert packet.status == :not_found + assert packet.key == "bar" + assert packet.value == "" + end + + test "getqk", %{conn: conn} do + :ok = Connection.send_packets(conn, [ + Packet.new(:getkq, key: "foo"), + Packet.new(:getk, key: "bar") + ]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :getk + assert packet.status == :not_found + assert packet.key == "bar" + assert packet.value == "" + end + + end + + describe "retrieving (hits)" do + + setup :flush + + setup %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:set, key: "foo", value: "bar")]) + {:ok, [%{opcode: :set}]} = Connection.recv_packets(conn, 1) + [conn: conn] + end + + test "get", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:get, key: "foo")]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :get + assert packet.status == :ok + assert packet.key == "" + assert packet.value == "bar" + end + + test "getq", %{conn: conn} do + :ok = Connection.send_packets(conn, [ + Packet.new(:getq, key: "baz"), + Packet.new(:getq, key: "foo") + ]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :getq + assert packet.status == :ok + assert packet.key == "" + assert packet.value == "bar" + end + + test "getk", %{conn: conn} do + :ok = Connection.send_packets(conn, [Packet.new(:getk, key: "foo")]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :getk + assert packet.status == :ok + assert packet.key == "foo" + assert packet.value == "bar" + end + + test "getkq", %{conn: conn} do + :ok = Connection.send_packets(conn, [ + Packet.new(:getkq, key: "baz"), + Packet.new(:getkq, key: "foo") + ]) + {:ok, [packet]} = Connection.recv_packets(conn, 1) + + assert packet.opcode == :getkq + assert packet.status == :ok + assert packet.key == "foo" + assert packet.value == "bar" + end + + end + + describe "extras (flags)" do + setup :flush + + test "set", %{conn: conn} do + :ok = Connection.send_packets(conn, [ + Packet.new(:set, key: "foo", value: "bar", flags: 123), + Packet.new(:get, key: "foo") + ]) + {:ok, [_, packet]} = Connection.recv_packets(conn, 2) + + assert packet.opcode == :get + assert packet.status == :ok + assert packet.value == "bar" + assert packet.extra.flags == 123 + end + + end + + def flush(%{conn: conn}) do + :ok = Connection.send_packets(conn, [Packet.new(:flush)]) + {:ok, [%{opcode: :flush}]} = Connection.recv_packets(conn, 1) + :ok + end + +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 83e89e5..869559e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1 @@ ExUnit.start() -{:ok, _} = Test.Cluster.start_link From 87aa4692e8d9255a53313eb9d988b7ca823ee500 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Thu, 19 Nov 2020 12:02:12 -0600 Subject: [PATCH 03/28] Start multi APIs --- lib/cream/connection.ex | 83 +++++++++++++++++++++++++++++++--------- lib/cream/packet.ex | 44 +++++++++++++-------- test/connection_test.exs | 36 +++++++++++++++++ 3 files changed, 130 insertions(+), 33 deletions(-) create mode 100644 test/connection_test.exs diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 3875f06..f185275 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -20,29 +20,25 @@ defmodule Cream.Connection do Connection.call(conn, {:recv_packets, count}) end - def get(conn, key, options \\ []) do - args = Keyword.merge(options, key: key) - with :ok <- send_packets(conn, [Packet.new(:get, args)]), + def flush(conn, options \\ []) do + with :ok <- send_packets(conn, [Packet.new(:flush, options)]), {:ok, [packet]} <- recv_packets(conn, 1) do - case packet.status do - :ok -> packet.value - :not_found -> nil - end - else - error -> error + packet.status end end - def set(conn, key, value, options \\ []) do - args = Keyword.merge(options, key: key, value: value) - with :ok <- send_packets(conn, [Packet.new(:set, args)]), - {:ok, [packet]} <- recv_packets(conn, 1) - do - packet.status - else - error -> error - end + def get(conn, key, options \\ []) do + args = Keyword.put(options, :key, key) + Connection.call(conn, {:get, args}) + end + + def set(conn, item, options \\ []) do + Connection.call(conn, {:set, item, options}) + end + + def mget(conn, keys, options \\ []) do + Connection.call(conn, {:mget, keys, options}) end def init(options) do @@ -74,6 +70,57 @@ defmodule Cream.Connection do {:reply, do_recv_packets(state.socket, count), state} end + def handle_call({:get, args}, _from, state) do + retval = with :ok <- do_send_packets(state.socket, [Packet.new(:get, args)]), + {:ok, [packet]} <- do_recv_packets(state.socket, 1) + do + cond do + packet.status == :not_found -> nil + args[:cas] -> {:ok, {packet.value, packet.cas}} + true -> {:ok, packet.value} + end + end + + {:reply, retval, state} + end + + def handle_call({:set, item, args}, _from, state) do + {key, value, cas} = case item do + {key, value} -> {key, value, 0} + {key, value, cas} -> {key, value, cas} + end + + args = Keyword.merge(args, [key: key, value: value, cas: cas]) + + retval = with :ok <- do_send_packets(state.socket, [Packet.new(:set, args)]), + {:ok, [packet]} <- do_recv_packets(state.socket, 1) + do + packet.status + end + + {:reply, retval, state} + end + + def handle_call({:mget, keys, options}, _from, state) do + packets = Enum.map(keys, fn key -> + Packet.new(:getkq, Keyword.put(options, :key, key)) + end) + + # Eww, no good way to append to list. + packets = packets ++ [Packet.new(:noop)] + + retval = with :ok <- do_send_packets(state.socket, packets), + {:ok, packets} <- do_recv_packets(state.socket, :noop) + do + Enum.reduce(packets, %{}, fn + packet, results when packet.opcode == :noop -> results + packet, results -> Map.put(results, packet.key, packet.value) + end) + end + + {:reply, retval, state} + end + defp do_send_packets(socket, packets) do Enum.reduce_while(packets, :ok, fn packet, _result -> case Socket.Stream.send(socket, Packet.serialize(packet)) do diff --git a/lib/cream/packet.ex b/lib/cream/packet.ex index 6b596f6..0b71100 100644 --- a/lib/cream/packet.ex +++ b/lib/cream/packet.ex @@ -4,19 +4,19 @@ defmodule Cream.Packet do @response_magic 0x81 defstruct [ - magic: @request_magic, - opcode: nil, - key_length: 0x0000, + magic: @request_magic, + opcode: nil, + key_length: 0x0000, extra_length: 0x00, - data_type: 0x00, - vbucket_id: 0x0000, - status: 0x0000, - total_body_length: 0x00000000, - opaque: 0x00000000, - cas: 0x0000000000000000, + data_type: 0x00, + vbucket_id: 0x0000, + status: 0x0000, + total_body_length: 0x00000000, + opaque: 0x00000000, + cas: 0x0000000000000000, extra: "", - key: "", - value: "" + key: "", + value: "" ] @atom_to_opcode %{ @@ -70,9 +70,11 @@ defmodule Cream.Packet do packet = %{packet | extra: serialize_extra(opcode, options), key: options[:key] || "", - value: options[:value] || "" + value: options[:value] || "", } + packet = add_cas(packet, options) + extra_length = IO.iodata_length(packet.extra) key_length = byte_size(packet.key) value_length = byte_size(packet.value) @@ -84,6 +86,18 @@ defmodule Cream.Packet do } end + def add_cas(packet, options) when packet.opcode in [:set, :setq] do + if options[:cas] do + %{packet | cas: options[:cas] } + else + packet + end + end + + def add_cas(packet, _options) do + packet + end + def serialize_extra(opcode, options) when opcode in [:set, :setq] do flags = options[:flags] || 0 expiration = options[:expiration] || 0 @@ -112,7 +126,7 @@ defmodule Cream.Packet do <>, <>, <>, - <>, + <>, <>, <>, <>, @@ -129,7 +143,7 @@ defmodule Cream.Packet do @response_magic :: bytes(1), opcode :: bytes(1), key_length :: bytes(2), - extra_length :: bytes(1), + extra_length :: bytes(1), data_type :: bytes(1), status :: bytes(2), total_body_length :: bytes(4), @@ -141,7 +155,7 @@ defmodule Cream.Packet do magic: @response_magic, opcode: @opcode_to_atom[opcode], key_length: key_length, - extra_length: extra_length, + extra_length: extra_length, data_type: data_type, status: @status_to_atom[status], total_body_length: total_body_length, diff --git a/test/connection_test.exs b/test/connection_test.exs new file mode 100644 index 0000000..a0bedd1 --- /dev/null +++ b/test/connection_test.exs @@ -0,0 +1,36 @@ +defmodule ConnectionTest do + use ExUnit.Case + + alias Cream.Connection + + setup_all do + {:ok, conn} = Connection.start_link() + [conn: conn] + end + + setup %{conn: conn} do + :ok = Connection.flush(conn) + end + + test "set/get", %{conn: conn} do + nil = Connection.get(conn, "foo") + nil = Connection.get(conn, "foo", cas: true) + + :ok = Connection.set(conn, {"foo", "bar"}) + + {:ok, "bar"} = Connection.get(conn, "foo") + {:ok, {"bar", cas}} = Connection.get(conn, "foo", cas: true) + + :exists = Connection.set(conn, {"foo", "baz", cas-1}) + {:ok, "bar"} = Connection.get(conn, "foo") + :ok = Connection.set(conn, {"foo", "baz", cas}) + {:ok, "baz"} = Connection.get(conn, "foo") + end + + test "mset/mget", %{conn: conn} do + :ok = Connection.set(conn, {"foo", "bar"}) + + Connection.mget(conn, ["foo", "bar"]) + end + +end From c1458cb56a9233819605f7a5513f27ef7c7a6389 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Tue, 8 Jun 2021 13:54:03 -0500 Subject: [PATCH 04/28] Basic connection --- lib/cream/application.ex | 8 +- lib/cream/cluster.ex | 467 -------------------------------- lib/cream/cluster/worker.ex | 121 --------- lib/cream/connection.ex | 318 +++++++++++++++------- lib/cream/instrumentation.ex | 53 ---- lib/cream/logger.ex | 42 +++ lib/cream/packet.ex | 182 ------------- lib/cream/protocol.ex | 235 ++++++++++++++++ lib/cream/registry.ex | 12 - lib/cream/supervisor/cluster.ex | 51 ---- lib/cream/utils.ex | 18 -- mix.exs | 3 +- mix.lock | 19 +- test/connection_test.exs | 44 ++- test/support/cluster.ex | 3 - 15 files changed, 536 insertions(+), 1040 deletions(-) delete mode 100644 lib/cream/cluster/worker.ex delete mode 100644 lib/cream/instrumentation.ex create mode 100644 lib/cream/logger.ex delete mode 100644 lib/cream/packet.ex create mode 100644 lib/cream/protocol.ex delete mode 100644 lib/cream/registry.ex delete mode 100644 lib/cream/supervisor/cluster.ex delete mode 100644 lib/cream/utils.ex delete mode 100644 test/support/cluster.ex diff --git a/lib/cream/application.ex b/lib/cream/application.ex index 08b9a24..1238776 100644 --- a/lib/cream/application.ex +++ b/lib/cream/application.ex @@ -1,14 +1,12 @@ defmodule Cream.Application do @moduledoc false - + use Application def start(_, _) do - import Supervisor.Spec, warn: false + :ok = Cream.Logger.init() - children = [ - worker(Registry, [:unique, Cream.Registry]), - ] + children = [] Supervisor.start_link(children, strategy: :one_for_one) end diff --git a/lib/cream/cluster.ex b/lib/cream/cluster.ex index 82c4185..151ddc8 100644 --- a/lib/cream/cluster.ex +++ b/lib/cream/cluster.ex @@ -1,470 +1,3 @@ defmodule Cream.Cluster do - @moduledoc """ - Connect to a cluster of memcached servers (or a single server if you want). - - ```elixir - {:ok, cluster} = Cream.Cluster.start_link(servers: ["host1:11211", "host2:11211"]) - Cream.Cluster.get(cluster, "foo") - ``` - - Using a module and Mix.Config is preferred... - ```elixir - # In config/*.exs - - use Mix.Config - config :my_app, MyCluster, - servers: ["host1:11211", "host2:11211"] - - # Elsewhere - - defmodule MyCluster do - use Cream.Cluster, otp_app: :my_app - end - - {:ok, _} = MyCluster.start_link - MyCluster.get("foo") - ``` - """ - - import Cream.Instrumentation - - @typedoc """ - Type representing a `Cream.Cluster`. - """ - @type t :: GenServer.server - - @typedoc """ - A memcached key. - """ - @type key :: String.t - - @typedoc """ - A list of keys. - """ - @type keys :: [key] - - @typedoc """ - A value to be stored in memcached. - """ - @type value :: String.t | serializable - - @typedoc """ - A key value pair. - """ - @type item :: {key, value} - - @typedoc """ - Multiple items as a list of tuples or a map. - """ - @type items :: [item] | [%{required(key) => value}] - - @typedoc """ - Reason associated with an error. - """ - @type reason :: String.t - - @typedoc """ - A `Memcache.Connection`. - """ - @type memcache_connection :: GenServer.server - - @typedoc """ - Anything serializable (to JSON). - """ - @type serializable :: list | map - - @typedoc """ - Configuration options. - """ - @type config :: Keyword.t - - defmacro __using__(opts) do - quote location: :keep, bind_quoted: [opts: opts] do - - @otp_app opts[:otp_app] - - def init(config), do: {:ok, config} - defoverridable [init: 1] - - def start_link(config \\ []) do - Cream.Cluster.start_link(__MODULE__, @otp_app, config) - end - - def child_spec(config) do - %{ id: __MODULE__, start: {__MODULE__, :start_link, [config]} } - end - - def set(items, opts \\ []), do: Cream.Cluster.set(__MODULE__, items, opts) - def get(key_or_keys, opts \\ []), do: Cream.Cluster.get(__MODULE__, key_or_keys, opts) - def delete(key_or_keys), do: Cream.Cluster.delete(__MODULE__, key_or_keys) - def fetch(key_or_keys, opts \\ [], func), do: Cream.Cluster.fetch(__MODULE__, key_or_keys, opts, func) - def with_conn(key_or_keys, func), do: Cream.Cluster.with_conn(__MODULE__, key_or_keys, func) - def flush(opts \\ []), do: Cream.Cluster.flush(__MODULE__, opts) - def put(key, value, opts \\ []), do: Cream.Cluster.put(__MODULE__, key, value, opts) - - end - end - - @doc """ - For dynamic / runtime configuration. - - Ex: - ```elixir - defmodule MyCluster do - use Cream.Cluster, otp_app: :my_app - - def init(config) do - servers = System.get_env("MEMCACHED_SERVERS") |> String.split(",") - config = Keyword.put(config, :servers, servers) - {:ok, config} - end - end - ``` - """ - @callback init(config) :: {:ok, config} | {:error, reason} - - @doc """ - For easily putting into supervision tree. - - Ex: - ```elixir - Supervisor.start_link([MyCluster], opts) - ``` - Or if you want to do runtime config here instead of the `c:init/1` callback for - some reason: - ```elixir - Supervisor.start_link([{MyCluster, servers: servers}], opts) - ``` - """ - @callback child_spec(config) :: Supervisor.child_spec - - @doc """ - See `set/3`. - """ - @callback set(item_or_items :: item | items, opts :: Keyword.t) :: - :ok | {:error, reason} - | %{required(key) => :ok | {:error | reason}} - - @doc """ - See `put/4`. - """ - @callback put(key, value, opts :: Keyword.t) :: :ok | {:error, reason} - - @doc """ - See `get/2`. - """ - @callback get(key_or_keys :: key | keys) :: value | items - - @doc """ - See `fetch/4`. - """ - @callback fetch(key_or_keys :: key | keys, f :: (() -> value) | (keys -> [value] | items)) :: value | items - - @doc """ - See `delete/2`. - """ - @callback delete(key_or_keys :: key | keys) :: - (:ok | {:error, reason}) - | [{key, :ok | {:error, reason}}] - - @doc """ - See `flush/2`. - """ - @callback flush(opts :: Keyword.t) :: :ok | {:error, reason} - - @doc """ - Connect to memcached server(s) - - ## Options - - * `:servers` - Servers to connect to. Defaults to `["localhost:11211"]`. - * `:pool` - Worker pool size. Defaults to `10`. - * `:name` - Like name argument for `GenServer.start_link/3`. No default. - Ignored if using module based cluster. - * `:memcachex` - Keyword list passed through to `Memcache.start_link/2` - - ## Example - - ```elixir - {:ok, cluster} = Cream.Cluster.start_link( - servers: ["host1:11211", "host2:11211"], - name: MyCluster, - memcachex: [ttl: 60, namespace: "foo"] - ) - ``` - """ - @defaults [ - servers: ["localhost:11211"], - pool: 5, - log: :debug - ] - @spec start_link(Keyword.t) :: t - def start_link(opts \\ []) do - opts = @defaults - |> Keyword.merge(opts) - |> Keyword.update!(:servers, &Cream.Utils.normalize_servers/1) - - poolboy_config = [ - worker_module: Cream.Supervisor.Cluster, - size: opts[:pool], - ] - - poolboy_config = if opts[:name] do - Keyword.put(poolboy_config, :name, {:local, opts[:name]}) - else - poolboy_config - end - - # This is so gross. There is no way to link or register a poolboy process, - # so we gotta wrap it in a task which subscribes to instrumentation. - - parent = self() - rand = :crypto.strong_rand_bytes(8) - - result = Task.start_link fn -> - if log_level = opts[:log] do - Instrumentation.subscribe "cream", fn tag, payload -> - Logger.bare_log(log_level, "cream.#{tag} #{inspect(payload)}") - end - end - - {:ok, _pid} = :poolboy.start_link(poolboy_config, opts) - send(parent, {:cream_ready, rand}) - :timer.sleep(:infinity) - end - - receive do - {:cream_ready, ^rand} -> result - after - 5000 -> {:error, "startup timeout"} - end - - end - - @doc false - # This is for starting a module based cluster. - def start_link(mod, otp_app, opts) do - opts = Keyword.put(opts, :name, mod) - config = Application.get_env(otp_app, mod) - with {:ok, config} <- mod.init(config) do - Keyword.merge(config, opts) |> start_link - end - end - - @doc ~S""" - Set the value of a single key. - - This a convenience function for `set(cluster, {key, value}, opts)`. - See `set/3`. - - It has a different name because the follow definitions conflict: - ```elixir - set(cluster, key, value, opts \\ []) - set(cluster, item, opts \\ []) - ``` - """ - @spec put(t, key, value, Keyword.t) :: :ok | {:error, reason} - def put(cluster, key, value, opts \\ []) do - set(cluster, {key, value}, opts) - end - - @doc """ - Set one or more keys. - - Single key examples: - ```elixir - set(cluster, {key, value}) - set(cluster, {key, value}, ttl: 300) - ``` - - Multiple key examples: - ```elixir - set(cluster, [{k1, v1}, {k2, v2}]) - set(cluster, [{k1, v1}, {k2, v2}], ttl: 300) - - set(cluster, %{k1 => v1, k2 => v2}) - set(cluster, %{k1 => v1, k2 => v2}, ttl: 300) - ``` - """ - @spec set(t, item | items, Keyword.t) :: :ok | {:error, reason} - def set(cluster, item_or_items, opts \\ []) - - def set(cluster, items, opts) when is_list(items) or is_map(items) do - with_worker cluster, fn worker -> - instrument "set", [items: items], fn -> - GenServer.call(worker, {:set, items, opts}) - end - end - end - - def set(cluster, item, opts) when is_tuple(item) do - set(cluster, [item], opts) |> Map.values |> List.first - end - - @doc """ - Get one or more keys. - - ## Examples - ``` - "bar" = get(pid, "foo") - %{"foo" => "bar", "one" => "one"} = get(pid, ["foo", "bar"]) - ``` - """ - @spec get(t, key, Keyword.t) :: value - @spec get(t, keys, Keyword.t) :: items - def get(cluster, key_or_keys, opts \\ []) - - def get(cluster, key, opts) when is_binary(key) do - case get(cluster, [key], opts) do - %{ ^key => value } -> value - _ -> nil - end - end - - def get(cluster, keys, opts) do - with_worker cluster, fn worker -> - instrument "get", [keys: keys], fn -> - GenServer.call(worker, {:get, keys, opts}) - end - end - end - - @doc """ - Fetch one or more keys, falling back to a function if a key doesn't exist. - - `opts` is the same as for `set/3`. - - Ex: - ```elixir - fetch(cluster, "foo", fn -> "bar" end) - - fetch(cluster, ["foo", "bar"], fn missing_keys -> - Enum.map(missing_keys, fn missing_key -> - calc_value(missing_key) - end) - end) - - # In this example, we explicitly associate missing keys with values. - fetch(cluster, ["foo", "bar"], fn missing_keys -> - Enum.shuffle(missing_keys) - |> Enum.map(fn missing_key -> - {missing_key, calc_value(missing_key)} - end) - end) - ``` - """ - @spec fetch(t, key, Keyword.t, (() -> value)) :: value - @spec fetch(t, keys, Keyword.t, (keys -> [value] | items)) :: items - def fetch(cluster, key_or_keys, opts \\ [], func) - - def fetch(cluster, key, opts, func) when is_binary(key) do - instrument "fetch", [key: key], fn -> - case get(cluster, [key], opts) do - %{^key => value} -> - {value, :hit} - %{} -> - value = func.() - set(cluster, {key, value}, opts) - {value, :miss} - end - end - end - - def fetch(cluster, keys, opts, func) when is_list(keys) do - instrument "fetch", [keys: keys], fn -> - hits = get(cluster, keys, opts) - missing_keys = Enum.reject(keys, &Map.has_key?(hits, &1)) - missing_hits = generate_missing(missing_keys, func) - set(cluster, missing_hits, opts) - results = Map.merge(hits, missing_hits) - {results, missing_keys} - end - end - - @doc """ - Delete one or more keys. - - ## Examples - ``` - delete("foo") - delete(["foo", "one"]) - ``` - """ - @spec delete(t, key) :: :ok | {:error, reason} - @spec delete(t, keys) :: %{required(key) => :ok | {:error, reason}} - def delete(cluster, key_or_keys) - - def delete(cluster, keys) when is_list(keys) do - with_worker cluster, fn worker -> - instrument "delete", [keys: keys], fn -> - GenServer.call(worker, {:delete, keys}) - end - end - end - - def delete(cluster, key) when is_binary(key) do - delete(cluster, [key]) |> Map.values |> List.first - end - - @doc """ - Get `Memcache` connection(s) for one or more keys. - - `Memcachex` doesn't support clustering. Cream supports clustering, but doesn't - support the full memcached API. This function lets you do both. - - The connection yielded to the function can be used with all of the `Memcache` - and `Memcache.Connection` modules. - - ## Examples - ``` - with_conn cluster, keys, fn conn, keys -> - Memcache.multi_get(conn, keys) - end - ``` - """ - @spec with_conn(t, key, (memcache_connection -> any)) :: any - @spec with_conn(t, keys, (memcache_connection, keys -> any)) :: [any] - def with_conn(cluster, key_or_keys, func) - - def with_conn(cluster, key, func) when is_binary(key) do - with_conn(cluster, [key], func) |> List.first - end - - def with_conn(cluster, keys, func) when is_list(keys) do - with_worker cluster, fn worker -> - GenServer.call(worker, {:with_conn, keys, func}) - end - end - - @doc """ - Flush all memcached servers in the cluster. - """ - @spec flush(t, Keyword.t) :: [:ok | {:error, reason}] - def flush(cluster, opts \\ []) do - with_worker cluster, fn worker -> - instrument "flush", fn -> - GenServer.call(worker, {:flush, opts}) - end - end - end - - defp generate_missing([], _func), do: %{} - defp generate_missing(keys, func) do - values = func.(keys) - cond do - is_map(values) -> values - is_list(values) -> Enum.zip(keys, values) |> Enum.into(%{}) - end - end - - defp with_worker(cluster, func) do - :poolboy.transaction cluster, fn supervisor -> - supervisor - |> Supervisor.which_children - |> Enum.find(& elem(&1, 0) == Cream.Cluster.Worker ) - |> elem(1) - |> func.() - end - end end diff --git a/lib/cream/cluster/worker.ex b/lib/cream/cluster/worker.ex deleted file mode 100644 index bbcc44a..0000000 --- a/lib/cream/cluster/worker.ex +++ /dev/null @@ -1,121 +0,0 @@ -require Logger - -defmodule Cream.Cluster.Worker do - @moduledoc false - - use GenServer - - alias Cream.Continuum - - def start_link(connection_map) do - GenServer.start_link(__MODULE__, connection_map) - end - - def init(connection_map) do - Logger.debug "Starting: #{inspect connection_map}" - - continuum = connection_map - |> Map.keys - |> Continuum.new - - {:ok, %{continuum: continuum, connection_map: connection_map}} - end - - def handle_call({:set, pairs, options}, _from, state) do - - pairs_by_conn = Enum.group_by pairs, fn {key, _value} -> - find_conn(state, key) - end - - reply = Enum.reduce pairs_by_conn, %{}, fn {conn, pairs}, acc -> - {:ok, responses} = Memcache.multi_set(conn, pairs, options) - - Enum.zip(pairs, responses) - |> Enum.reduce(acc, fn {pair, status}, acc -> - {key, _value} = pair - status = case status do - {:ok} -> :ok - status -> status - end - Map.put(acc, key, status) - end) - end - - {:reply, reply, state} - end - - def handle_call({:get, keys, options}, _from, state) do - keys_by_conn = Enum.group_by keys, fn key -> - find_conn(state, key) - end - - reply = Enum.reduce keys_by_conn, %{}, fn {conn, keys}, acc -> - case Memcache.multi_get(conn, keys, options) do - {:ok, map} -> Map.merge(acc, map) - _ -> acc # TODO something better than silently ignore? - end - end - - {:reply, reply, state} - end - - def handle_call({:with_conn, keys, func}, _from, state) do - keys_by_conn_and_server = Enum.group_by keys, fn key -> - find_conn_and_server(state, key) - end - - keys_by_conn_and_server - |> Enum.reduce(%{}, fn {{conn, server}, keys}, acc -> - Map.put(acc, server, func.(conn, keys)) - end) - |> reply(state) - end - - def handle_call({:flush, options}, _from, state) do - ttl = Keyword.get(options, :ttl, 0) - - reply = Enum.map state.connection_map, fn {_server, conn} -> - case Memcache.Connection.execute(conn, :FLUSH, [ttl]) do - {:ok} -> :ok - whatever -> whatever - end - end - - {:reply, reply, state} - end - - def handle_call({:delete, keys}, _from, state) do - Enum.group_by(keys, &find_conn(state, &1)) - |> Enum.reduce(%{}, fn {conn, keys}, results -> - commands = Enum.map(keys, &{:DELETEQ, [&1]}) - case Memcache.Connection.execute_quiet(conn, commands) do - {:ok, conn_results} -> Enum.zip(keys, conn_results) - |> Enum.reduce(results, fn {key, conn_results}, results -> - case conn_results do - {:ok} -> Map.put(results, key, :ok) - error -> Map.put(results, key, error) - end - end) - {:error, _} -> Enum.reduce(keys, results, fn key, results -> - Map.put(results, key, {:error, "connection error"}) - end) - end - end) - |> reply(state) - end - - defp reply(reply, state) do - {:reply, reply, state } - end - - defp find_conn_and_server(state, key) do - {:ok, server} = Continuum.find(state.continuum, key) - conn = state.connection_map[server] - {conn, server} - end - - defp find_conn(state, key) do - find_conn_and_server(state, key) |> elem(0) - end - -end diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index f185275..d8ea27a 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -1,185 +1,295 @@ defmodule Cream.Connection do use Connection require Logger - alias Cream.Packet + alias Cream.{Protocol} @defaults [ server: "localhost:11211" ] - def start_link(options \\ []) do - options = Keyword.merge(@defaults, options) - Connection.start_link(__MODULE__, options) - end + def start_link(config \\ []) do + mix_config = Application.get_application(__MODULE__) + |> Application.get_env(__MODULE__, []) - def send_packets(conn, packets) do - Connection.call(conn, {:send_packets, packets}) - end + config = Keyword.merge(@defaults, mix_config) + |> Keyword.merge(config) - def recv_packets(conn, count) do - Connection.call(conn, {:recv_packets, count}) + Connection.start_link(__MODULE__, config) end def flush(conn, options \\ []) do - with :ok <- send_packets(conn, [Packet.new(:flush, options)]), - {:ok, [packet]} <- recv_packets(conn, 1) - do - packet.status - end + Connection.call(conn, {:flush, options}) + end + + def set(conn, item, options \\ []) do + Connection.call(conn, {:set, item, options}) end def get(conn, key, options \\ []) do - args = Keyword.put(options, :key, key) - Connection.call(conn, {:get, args}) + Connection.call(conn, {:get, key, options}) end - def set(conn, item, options \\ []) do - Connection.call(conn, {:set, item, options}) + def mset(conn, items, options \\ []) do + Connection.call(conn, {:mset, items, options}) end def mget(conn, keys, options \\ []) do Connection.call(conn, {:mget, keys, options}) end - def init(options) do + def noop(conn) do + Connection.call(conn, :noop) + end + + def init(config) do state = %{ - options: options, + config: config, socket: nil, + coder: nil, + errors: 0, } {:connect, :init, state} end - def connect(_context, state) do - server = state.options[:server] - url = "tcp://#{server}" + def connect(context, state) do + %{config: config} = state + + server = config[:server] + [host, port] = String.split(server, ":") + host = String.to_charlist(host) + port = String.to_integer(port) + + start = System.monotonic_time(:microsecond) + + case :gen_tcp.connect(host, port, [:binary, active: true]) do + {:ok, socket} -> + :telemetry.execute( + [:cream, :connection, :connect], + %{usec: System.monotonic_time(:microsecond) - start}, + %{context: context, server: server} + ) + {:ok, %{state | socket: socket, errors: 0}} - case Socket.connect(url) do - {:ok, socket} -> {:ok, %{state | socket: socket}} {:error, reason} -> - Logger.warn("#{server} #{reason}") - {:backoff, 1000, state} + errors = state.errors + 1 + :telemetry.execute( + [:cream, :connection, :error], + %{usec: System.monotonic_time(:microsecond) - start}, + %{context: context, reason: reason, server: server, count: errors} + ) + {:backoff, 1000, %{state | errors: errors}} end end - def handle_call({:send_packets, packets}, _from, state) do - {:reply, do_send_packets(state.socket, packets), state} + def disconnect(reason, state) do + %{config: config, socket: socket} = state + + :telemetry.execute( + [:cream, :connection, :disconnect], + %{}, + %{reason: reason, server: config[:server]} + ) + + :ok = :gen_tcp.close(socket) + + {:connect, reason, %{state | socket: nil}} end - def handle_call({:recv_packets, count}, _from, state) do - {:reply, do_recv_packets(state.socket, count), state} + def handle_call(_command, _from, state) when is_nil(state.socket) do + {:reply, {:error, :not_connected}, state} end - def handle_call({:get, args}, _from, state) do - retval = with :ok <- do_send_packets(state.socket, [Packet.new(:get, args)]), - {:ok, [packet]} <- do_recv_packets(state.socket, 1) + def handle_call({:flush, options}, _from, state) do + %{socket: socket} = state + + with :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, Protocol.flush(options)), + {:ok, packet} <- Protocol.recv_packet(socket), + :ok <- :inet.setopts(socket, active: true) do - cond do - packet.status == :not_found -> nil - args[:cas] -> {:ok, {packet.value, packet.cas}} - true -> {:ok, packet.value} + case packet.status do + :ok -> {:reply, :ok, state} + error -> {:reply, {:error, error}, state} end + else + {:error, reason} -> {:disconnect, reason, state} end - - {:reply, retval, state} end - def handle_call({:set, item, args}, _from, state) do - {key, value, cas} = case item do - {key, value} -> {key, value, 0} - {key, value, cas} -> {key, value, cas} + def handle_call({:set, item, options}, _from, state) do + %{socket: socket} = state + + with {:ok, key, value, item_opts} <- parse_item(item), + {:ok, value, flags} <- encode(value, state), + packet = Protocol.set(key, value, flags, item_opts), + :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, packet), + {:ok, packet} <- Protocol.recv_packet(socket), + :ok <- :inet.setopts(socket, active: true) + do + case packet.status do + :ok -> if options[:cas] do + {:reply, {:ok, packet.cas}, state} + else + {:reply, :ok, state} + end + error -> {:reply, {:error, error}, state} + end + else + {:error, reason} -> {:disconnect, reason, state} end + end + + def handle_call({:get, key, options}, _from, state) do + %{socket: socket} = state - args = Keyword.merge(args, [key: key, value: value, cas: cas]) + packet = Protocol.get(key, options) - retval = with :ok <- do_send_packets(state.socket, [Packet.new(:set, args)]), - {:ok, [packet]} <- do_recv_packets(state.socket, 1) + with :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, packet), + {:ok, packet} <- Protocol.recv_packet(socket), + :ok <- :inet.setopts(socket, active: true) do - packet.status + case packet.status do + :ok -> + {:ok, value} = decode(packet.value, packet.extras.flags, state) + case options[:cas] do + true -> {:reply, {:ok, {value, packet.cas}}, state} + _ -> {:reply, {:ok, value}, state} + end + error -> {:reply, {:error, error}, state} + end + else + {:error, reason} -> {:disconnect, reason, state} end + end + + def handle_call({:mset, items, options}, _from, state) do + %{socket: socket} = state + + packet_opts = Keyword.take(options, [:expiry]) + + with {:ok, packets} <- items_to_packets(items, packet_opts, state), + :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, packets), + {:ok, packets} <- Protocol.recv_packets(socket, Enum.count(packets)), + :ok <- :inet.setopts(socket, active: true) + do + results = if options[:cas] do + Enum.map(packets, fn + %{status: :ok, cas: cas} -> {:ok, cas} + %{status: reason} -> {:error, reason} + end) + else + Enum.map(packets, fn + %{status: :ok} -> :ok + %{status: reason} -> {:error, reason} + end) + end - {:reply, retval, state} + {:reply, {:ok, results}, state} + else + {:error, reason} -> {:reply, {:error, reason}, state} + end end def handle_call({:mget, keys, options}, _from, state) do - packets = Enum.map(keys, fn key -> - Packet.new(:getkq, Keyword.put(options, :key, key)) + %{socket: socket} = state + + packets = Enum.map(keys, fn + {key, _opts} -> Protocol.getkq(to_string(key)) + key -> Protocol.getkq(to_string(key)) end) - # Eww, no good way to append to list. - packets = packets ++ [Packet.new(:noop)] + packets = [packets, Protocol.noop()] - retval = with :ok <- do_send_packets(state.socket, packets), - {:ok, packets} <- do_recv_packets(state.socket, :noop) + with :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, packets), + {:ok, packets} <- Protocol.recv_packets(socket, :noop), + :ok <- :inet.setopts(socket, active: true) do - Enum.reduce(packets, %{}, fn - packet, results when packet.opcode == :noop -> results - packet, results -> Map.put(results, packet.key, packet.value) + values_by_key = Enum.reduce_while(packets, %{}, fn packet, acc -> + case packet.opcode do + :noop -> {:halt, acc} + _ -> {:cont, Map.put(acc, packet.key, {packet.value, packet.cas})} + end end) - end - {:reply, retval, state} - end + cas = Keyword.get(options, :cas, false) - defp do_send_packets(socket, packets) do - Enum.reduce_while(packets, :ok, fn packet, _result -> - case Socket.Stream.send(socket, Packet.serialize(packet)) do - :ok -> {:cont, :ok} - error -> {:halt, error} - end - end) + responses = Enum.map(keys, fn key -> + {key, opts} = case key do + {key, opts} -> {key, opts} + key -> {key, []} + end + + cas = opts[:cas] || cas + + case {values_by_key[key], cas} do + {nil, _} -> {:error, :not_found} + {{value, _cas}, false} -> {:ok, value} + {value_cas, true} -> {:ok, value_cas} + end + end) + + {:reply, {:ok, responses}, state} + else + {:error, reason} -> {:disconnect, reason, state} + end end - defp do_recv_packets(socket, :noop) do - Stream.repeatedly(fn -> do_recv_packet(socket) end) - |> Enum.reduce_while([], fn - {:ok, packet}, packets -> if packet.opcode == :noop do - {:halt, [packet | packets]} - else - {:cont, [packet | packets]} - end - error, _packets -> {:halt, error} - end) - |> case do - {:error, reason} -> {:error, reason} - packets -> {:ok, Enum.reverse(packets)} + def handle_call(:noop, _from, state) do + %{socket: socket} = state + + with :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, Protocol.noop()), + {:ok, packet} <- Protocol.recv_packet(socket), + :ok <- :inet.setopts(socket, active: true) + do + {:reply, {:ok, packet}, state} + else + error -> {:reply, error, state} end end - defp do_recv_packets(socket, count) do - Enum.reduce_while(1..count, [], fn _i, packets -> - case do_recv_packet(socket) do - {:ok, packet} -> {:cont, [packet | packets]} + def handle_info({:tcp_closed, _socket}, state) do + {:disconnect, :tcp_closed, state} + end + + defp items_to_packets(items, opts, state) do + Enum.reduce_while(items, {:ok, []}, fn item, {:ok, packets} -> + with {:ok, key, value, opts} <- parse_item(item, opts), + {:ok, value, flags} <- encode(value, state) + do + packet = Protocol.set(key, value, flags, opts) + {:cont, {:ok, [packet | packets]}} + else error -> {:halt, error} end end) |> case do - {:error, reason} -> {:error, reason} - packets -> {:ok, Enum.reverse(packets)} - end - end - - defp do_recv_packet(socket) do - with {:ok, data} <- do_recv_header(socket), - packet = Packet.deserialize_header(data), - {:ok, data} <- do_recv_body(socket, packet) - do - {:ok, Packet.deserialize_body(packet, data)} - else + {:ok, packets} -> {:ok, Enum.reverse(packets)} error -> error end end - defp do_recv_header(socket) do - Socket.Stream.recv(socket, 24) + defp parse_item(item, options \\ []) do + case item do + {key, value} -> {:ok, key, value, options} + {key, value, opts} -> {:ok, key, value, Keyword.merge(options, opts)} + _ -> {:error, :invalid_item} + end end - defp do_recv_body(_socket, packet) when packet.total_body_length == 0 do - {:ok, ""} + defp encode(value, %{coder: nil}), do: {:ok, value, 0} + defp encode(value, %{coder: coder}) do + coder.encode(value) end - defp do_recv_body(socket, packet) when packet.total_body_length > 0 do - Socket.Stream.recv(socket, packet.total_body_length) + defp decode(value, _flags, %{coder: nil}), do: {:ok, value} + defp decode(value, flags, %{coder: coder}) do + coder.decode(value, flags) end end diff --git a/lib/cream/instrumentation.ex b/lib/cream/instrumentation.ex deleted file mode 100644 index a97aabd..0000000 --- a/lib/cream/instrumentation.ex +++ /dev/null @@ -1,53 +0,0 @@ -defmodule Cream.Instrumentation do - @moduledoc false - - def instrument(tag, opts \\ [], f) do - Instrumentation.instrument "cream", tag, fn -> - f.() |> payload(tag, Map.new(opts)) - end - end - - defp payload(results, "set", %{items: items}) do - keys = Enum.map(items, fn {k, _} -> k end) - {results, [keys: keys]} - end - - defp payload(results, "get", %{keys: keys}) do - misses = Enum.reject(keys, fn key -> Map.has_key?(results, key) end) - {results, keys: keys, misses: misses} - end - - defp payload(wrapped, "fetch", %{key: key}) do - case wrapped do - {results, :hit} -> {results, keys: [key], misses: []} - {results, :miss} -> {results, keys: [key], misses: [key]} - end - end - - defp payload(wrapped, "fetch", %{keys: keys}) do - {results, misses} = wrapped - {results, keys: keys, misses: misses} - end - - defp payload(results, "delete", %{keys: keys}) do - misses = Enum.filter(keys, fn key -> - case results[key] do - {:error, _} -> true - _ -> false - end - end) - {results, keys: keys, misses: misses} - end - - defp payload(results, "flush", %{}) do - {successes, failures} = Enum.reduce(results, {0,0}, fn result, {s, f} -> - case result do - :ok -> {s+1, f} - _ -> {s, f+1} - end - end) - - {results, [successes: successes, failures: failures]} - end - -end diff --git a/lib/cream/logger.ex b/lib/cream/logger.ex new file mode 100644 index 0000000..28d2121 --- /dev/null +++ b/lib/cream/logger.ex @@ -0,0 +1,42 @@ +defmodule Cream.Logger do + require Logger + + def init do + config = Application.get_env(:cream, __MODULE__, []) + |> Map.new() + + :telemetry.attach_many( + inspect(__MODULE__), + [ + [:cream, :connection, :connect], + [:cream, :connection, :error], + [:cream, :connection, :disconnect] + ], + &__MODULE__.log/4, + config + ) + end + + def log([:cream, :connection, :connect], %{usec: usec}, meta, _config) do + time = format_usec(usec) + Logger.debug("Connected to #{meta.server} in #{time}") + end + + def log([:cream, :connection, :error], %{usec: usec}, meta, _config) do + time = format_usec(usec) + Logger.warn("Error connecting to #{meta.server} (#{meta.reason}) in #{time}") + end + + def log([:cream, :connection, :disconnect], _time, meta, _config) do + Logger.warn("Disconnected from #{meta.server} (#{meta.reason})") + end + + def format_usec(usec) do + if usec < 1000 do + "#{usec}μs" + else + "#{round(usec/1000)}ms" + end + end + +end diff --git a/lib/cream/packet.ex b/lib/cream/packet.ex deleted file mode 100644 index 0b71100..0000000 --- a/lib/cream/packet.ex +++ /dev/null @@ -1,182 +0,0 @@ -defmodule Cream.Packet do - - @request_magic 0x80 - @response_magic 0x81 - - defstruct [ - magic: @request_magic, - opcode: nil, - key_length: 0x0000, - extra_length: 0x00, - data_type: 0x00, - vbucket_id: 0x0000, - status: 0x0000, - total_body_length: 0x00000000, - opaque: 0x00000000, - cas: 0x0000000000000000, - extra: "", - key: "", - value: "" - ] - - @atom_to_opcode %{ - get: 0x00, - set: 0x01, - flush: 0x08, - getq: 0x09, - noop: 0x0a, - getk: 0x0c, - getkq: 0x0d, - setq: 0x11, - } - - @atom_to_status %{ - ok: 0x0000, - not_found: 0x0001, - exists: 0x0002, - too_large: 0x0003, - invalid_args: 0x0004, - not_stored: 0x0005, - non_numeric: 0x0006, - vbucket_error: 0x0007, - auth_error: 0x0008, - auth_cont: 0x0009, - unknown_cmd: 0x0081, - oom: 0x0082, - not_supported: 0x0083, - internal_error: 0x0084, - busy: 0x0085, - temp_failure: 0x0086 - } - - @opcode_to_atom Enum.map(@atom_to_opcode, fn {k, v} -> {v, k} end) |> Map.new() - @status_to_atom Enum.map(@atom_to_status, fn {k, v} -> {v, k} end) |> Map.new() - - defmacrop bytes(n) do - bytes = n*8 - quote do: size(unquote(bytes)) - end - - def new(opcode, options \\ []) do - if not Map.has_key?(@atom_to_opcode, opcode) do - raise "unknown opcode #{opcode}" - end - - packet = %__MODULE__{ - magic: @request_magic, - opcode: opcode, - } - - packet = %{packet | - extra: serialize_extra(opcode, options), - key: options[:key] || "", - value: options[:value] || "", - } - - packet = add_cas(packet, options) - - extra_length = IO.iodata_length(packet.extra) - key_length = byte_size(packet.key) - value_length = byte_size(packet.value) - - %{packet | - extra_length: extra_length, - key_length: key_length, - total_body_length: extra_length + key_length + value_length - } - end - - def add_cas(packet, options) when packet.opcode in [:set, :setq] do - if options[:cas] do - %{packet | cas: options[:cas] } - else - packet - end - end - - def add_cas(packet, _options) do - packet - end - - def serialize_extra(opcode, options) when opcode in [:set, :setq] do - flags = options[:flags] || 0 - expiration = options[:expiration] || 0 - - [ - <>, - <>, - ] - end - - def serialize_extra(_opcode, _options), do: [] - - def deserialize_extra(_opcode, ""), do: %{} - - def deserialize_extra(opcode, data) when opcode in [:get, :getq, :getk, :getkq] do - <> = data - %{flags: flags} - end - - def deserialize_extra(_opcode, _data), do: %{} - - def serialize(packet) when packet.magic == @request_magic do - opcode = @atom_to_opcode[packet.opcode] - - [ - <>, - <>, - <>, - <>, - <>, - <>, - <>, - <>, - <>, - packet.extra, - packet.key, - packet.value - ] - end - - def deserialize_header(data) do - << - @response_magic :: bytes(1), - opcode :: bytes(1), - key_length :: bytes(2), - extra_length :: bytes(1), - data_type :: bytes(1), - status :: bytes(2), - total_body_length :: bytes(4), - opaque :: bytes(4), - cas :: bytes(8) - >> = data - - %__MODULE__{ - magic: @response_magic, - opcode: @opcode_to_atom[opcode], - key_length: key_length, - extra_length: extra_length, - data_type: data_type, - status: @status_to_atom[status], - total_body_length: total_body_length, - opaque: opaque, - cas: cas - } - end - - def deserialize_body(packet, data) do - extra_length = packet.extra_length - key_length = packet.key_length - - << - extra :: binary-size(extra_length), - key :: binary-size(key_length), - value :: binary - >> = data - - extra = deserialize_extra(packet.opcode, extra) - - %{packet | extra: extra, key: key, value: value} - end - -end diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex new file mode 100644 index 0000000..75444ab --- /dev/null +++ b/lib/cream/protocol.ex @@ -0,0 +1,235 @@ +defmodule Cream.Protocol do + + @request 0x80 + @response 0x81 + + @get 0x00 + @set 0x01 + @flush 0x08 + @getq 0x09 + @noop 0x0a + @getk 0x0c + @getkq 0x0d + @setq 0x11 + + [ + {0x0000, :ok}, + {0x0001, :not_found}, + {0x0002, :exists}, + {0x0003, :too_large}, + {0x0004, :invalid_args}, + {0x0005, :no_stored}, + {0x0006, :non_numeric}, + {0x0007, :vbucket_error}, + {0x0008, :auth_error}, + {0x0009, :auth_cont}, + {0x0081, :unknown_cmd}, + {0x0082, :oom}, + {0x0083, :not_supported}, + {0x0084, :internal_error}, + {0x0085, :busy}, + {0x0086, :temp_failure} + ] + |> Enum.each(fn {code, atom} -> + def status_to_atom(unquote(code)), do: unquote(atom) + end) + + defmacrop bytes(n) do + bytes = n*8 + quote do: size(unquote(bytes)) + end + + def noop do + [ + <<@request::bytes(1)>>, + <<@noop::bytes(1)>>, + <<0x00::bytes(2)>>, + <<0x00::bytes(1)>>, + <<0x00::bytes(1)>>, + <<0x00::bytes(2)>>, + <<0x00::bytes(4)>>, + <<0x00::bytes(4)>>, + <<0x00::bytes(8)>>, + ] + end + + def flush(options \\ []) do + expiry = Keyword.get(options, :expiry, 0) + + [ + <<@request::bytes(1)>>, + <<@flush::bytes(1)>>, + <<0x00::bytes(2)>>, + <<0x04::bytes(1)>>, + <<0x00::bytes(1)>>, + <<0x00::bytes(2)>>, + <<0x04::bytes(4)>>, + <<0x00::bytes(4)>>, + <<0x00::bytes(8)>>, + <> + ] + end + + def set(key, value, flags, options \\ []) do + expiry = Keyword.get(options, :expiry, 0) + cas = Keyword.get(options, :cas, 0) + + key_size = byte_size(key) + value_size = byte_size(value) + body_size = key_size + value_size + 8 + + [ + @request, + @set, + <>, + 0x08, + 0x00, + <<0x00::bytes(2)>>, + <>, + <<0x00::bytes(4)>>, + <>, + <>, + <>, + key, + value + ] + end + + def setq(key, value, flags, options \\ []) do + expiry = Keyword.get(options, :expiry, 0) + cas = Keyword.get(options, :cas, 0) + + key_size = byte_size(key) + value_size = byte_size(value) + body_size = key_size + value_size + 8 + + [ + @request, + @setq, + <>, + 0x08, + 0x00, + <<0x00::bytes(2)>>, + <>, + <<0x00::bytes(4)>>, + <>, + <>, + <>, + key, + value + ] + end + + def get(key, _options \\ []) do + key_size = byte_size(key) + + [ + @request, + @get, + <>, + 0x00, + 0x00, + <<0x00::size(16)>>, + <>, + <<0x00::size(32)>>, + <<0x00::size(64)>>, + key + ] + end + + def getkq(key, _options \\ []) do + key_size = byte_size(key) + + [ + @request, + @getkq, + <>, + 0x00, + 0x00, + <<0x00::size(16)>>, + <>, + <<0x00::size(32)>>, + <<0x00::size(64)>>, + key + ] + end + + def recv_packet(socket) do + with {:ok, header, body} <- recv_packet_data(socket) do + {:ok, parse_packet(header, body)} + end + end + + def recv_packets(socket, how, packets \\ []) + + def recv_packets(socket, :noop, packets) do + with {:ok, packet} <- recv_packet(socket) do + case packet do + %{opcode: @noop} -> {:ok, Enum.reverse(packets)} + packet -> recv_packets(socket, :noop, [packet | packets]) + end + end + end + + def recv_packets(_socket, 0, packets) do + {:ok, Enum.reverse(packets)} + end + + def recv_packets(socket, count, packets) when is_integer(count) do + with {:ok, packet} <- recv_packet(socket) do + recv_packets(socket, count-1, [packet | packets]) + end + end + + defp recv_packet_data(socket) do + with {:ok, header} <- :gen_tcp.recv(socket, 24) do + <<_::bytes(8), body_length::bytes(4), _::binary>> = header + if body_length == 0 do + {:ok, header, ""} + else + with {:ok, body} <- :gen_tcp.recv(socket, body_length) do + {:ok, header, body} + end + end + end + end + + defp parse_packet(header, body) do + << + @response, + opcode::bytes(1), + key_length::bytes(2), + extra_length::bytes(1), + _data_type::bytes(1), + status::bytes(2), + body_length::bytes(4), + _opaque::bytes(4), + cas::bytes(8), + >> = header + + value_length = body_length - extra_length - key_length + + << + extras::binary-size(extra_length), + key::binary-size(key_length), + value::binary-size(value_length) + >> = body + + %{ + opcode: opcode, + status: status_to_atom(status), + cas: cas, + key: key, + value: value, + extras: parse_extras(opcode, extras) + } + end + + defp parse_extras(_opcode, ""), do: %{} + defp parse_extras(opcode, extras) when opcode in [@get, @getq, @getk, @getkq] do + <> = extras + %{flags: flags} + end + defp parse_extras(_opcode, _extras), do: %{} + +end diff --git a/lib/cream/registry.ex b/lib/cream/registry.ex deleted file mode 100644 index 677b36d..0000000 --- a/lib/cream/registry.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Cream.Registry do - @moduledoc false - - def new_cluster do - {:via, Registry, {Cream.Registry, {Cream.Cluster, UUID.uuid4}}} - end - - def new_connection do - {:via, Registry, {Cream.Registry, {Memcache, UUID.uuid4}}} - end - -end diff --git a/lib/cream/supervisor/cluster.ex b/lib/cream/supervisor/cluster.ex deleted file mode 100644 index e48c7ff..0000000 --- a/lib/cream/supervisor/cluster.ex +++ /dev/null @@ -1,51 +0,0 @@ -defmodule Cream.Supervisor.Cluster do - @moduledoc false - - use Supervisor - - import Cream.Utils, only: [parse_server: 1] - - def start_link(options) do - Supervisor.start_link(__MODULE__, options) - end - - def init(options) do - servers = options[:servers] - - # Make a map of server to Memcache.Connection name so that - # Cluster.Worker process knows how to access the connections managed by - # this supervisor. - # Ex: - # %{ - # "127.0.0.1:11211" => {:via, Registry, ...}, - # "localhost:11211" => {:via, Registry, ...} - # } - server_name_map = Enum.reduce servers, %{}, fn server, acc -> - Map.put(acc, server, Cream.Registry.new_connection) - end - - # Pull out the memcachex specific options. - memcachex_options = Keyword.get(options, :memcachex, []) - - # Each Memcache worker gets supervised. - specs = Enum.map server_name_map, fn {server, name} -> - {host, port} = parse_server(server) - arguments = memcachex_options - |> Keyword.merge(hostname: host, port: port) - - worker( - Memcache, - [arguments, [name: name]], - id: {Memcache, server} - ) - end - - # Cluster.Worker gets supervised and passed the connection name map. - specs = [ - worker(Cream.Cluster.Worker, [server_name_map]) | specs - ] - - supervise(specs, strategy: :one_for_one) - end - -end diff --git a/lib/cream/utils.ex b/lib/cream/utils.ex deleted file mode 100644 index a10b293..0000000 --- a/lib/cream/utils.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Cream.Utils do - @moduledoc false - - def normalize_servers(servers) do - List.wrap(servers) |> Enum.map(fn server -> - case to_string(server) |> String.split(":") do - [host | []] -> "#{host}:11211" - [host, port | []] -> "#{host}:#{port}" - end - end) - end - - def parse_server(server) do - [host, port | []] = server |> to_string |> String.split(":") - {host, String.to_integer(port)} - end - -end diff --git a/mix.exs b/mix.exs index 901b054..8aace18 100644 --- a/mix.exs +++ b/mix.exs @@ -42,7 +42,7 @@ defmodule Cream.Mixfile do def application do [ # Specify extra applications you'll use from Erlang/Elixir - extra_applications: [:logger], + extra_applications: [:logger, :crypto], mod: {Cream.Application, []} ] end @@ -60,7 +60,6 @@ defmodule Cream.Mixfile do [ {:telemetry, "~> 0.4.2"}, {:connection, "~> 1.0"}, - {:socket, "~> 0.3.13"}, ] end end diff --git a/mix.lock b/mix.lock index 6427450..1ea0e43 100644 --- a/mix.lock +++ b/mix.lock @@ -1,19 +1,4 @@ %{ - "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, - "crc": {:hex, :crc, "0.5.2", "6db0c06f4bb2ae6a737a32b31fd40842774d4aae903b76e5f4dae44bd4b2742c", [:make, :mix], []}, - "db_connection": {:hex, :db_connection, "1.1.0", "b2b88db6d7d12f99997b584d09fad98e560b817a20dab6a526830e339f54cdb3", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, - "decimal": {:hex, :decimal, "1.3.1", "157b3cedb2bfcb5359372a7766dd7a41091ad34578296e951f58a946fcab49c6", [:mix], []}, - "earmark": {:hex, :earmark, "1.2.2", "f718159d6b65068e8daeef709ccddae5f7fdc770707d82e7d126f584cd925b74", [:mix], []}, - "ecto": {:hex, :ecto, "2.0.6", "9dcbf819c2a77f67a66b83739b7fcc00b71aaf6c100016db4f798930fa4cfd47", [:mix], [{:db_connection, "~> 1.0", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.1.2 or ~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.12.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0-beta", [hex: :sbroker, optional: true]}]}, - "ex_doc": {:hex, :ex_doc, "0.16.1", "b4b8a23602b4ce0e9a5a960a81260d1f7b29635b9652c67e95b0c2f7ccee5e81", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]}, - "instrumentation": {:hex, :instrumentation, "0.1.0", "31878b434e87402e5234ffcce2467a35687b977c7b2c8620f6346427d6462b76", [:mix], [], "hexpm"}, - "jiffy": {:hex, :jiffy, "0.14.11", "919a87d491c5a6b5e3bbc27fafedc3a0761ca0b4c405394f121f582fd4e3f0e5", [:rebar3], []}, - "memcache_client": {:hex, :memcache_client, "1.1.0", "aa643e9a4ce4987232d9f84867bb420ec4725c6625a01dfe09c7cec7a90db128", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poison, "~> 2.1.0", [hex: :poison, optional: false]}, {:poolboy, "~> 1.5.1", [hex: :poolboy, optional: false]}]}, - "memcachex": {:hex, :memcachex, "0.4.1", "12fe48370a5725fff790e1cdc0aa9ce2c28f685ad534bfbad5ef2302ee5003b6", [:mix], [{:connection, "~> 1.0.3", [hex: :connection, optional: false]}, {:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: true]}]}, - "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []}, - "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []}, - "postgrex": {:hex, :postgrex, "0.12.1", "2f8b46cb3a44dcd42f42938abedbfffe7e103ba4ce810ccbeee8dcf27ca0fb06", [:mix], [{:connection, "~> 1.0", [hex: :connection, optional: false]}, {:db_connection, "~> 1.0-rc.4", [hex: :db_connection, optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, optional: false]}]}, - "socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"}, - "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, - "uuid": {:hex, :uuid, "1.1.6", "4927232f244e69c6e255643014c2d639dad5b8313dc2a6976ee1c3724e6ca60d", [:mix], []}, + "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, + "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, } diff --git a/test/connection_test.exs b/test/connection_test.exs index a0bedd1..a1f00a4 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -13,24 +13,58 @@ defmodule ConnectionTest do end test "set/get", %{conn: conn} do - nil = Connection.get(conn, "foo") - nil = Connection.get(conn, "foo", cas: true) + {:error, :not_found} = Connection.get(conn, "foo") + {:error, :not_found} = Connection.get(conn, "foo", cas: true) :ok = Connection.set(conn, {"foo", "bar"}) {:ok, "bar"} = Connection.get(conn, "foo") {:ok, {"bar", cas}} = Connection.get(conn, "foo", cas: true) - :exists = Connection.set(conn, {"foo", "baz", cas-1}) + {:error, :exists} = Connection.set(conn, {"foo", "baz", cas: cas-1}) {:ok, "bar"} = Connection.get(conn, "foo") - :ok = Connection.set(conn, {"foo", "baz", cas}) + :ok = Connection.set(conn, {"foo", "baz", cas: cas}) {:ok, "baz"} = Connection.get(conn, "foo") end test "mset/mget", %{conn: conn} do :ok = Connection.set(conn, {"foo", "bar"}) - Connection.mget(conn, ["foo", "bar"]) + {:ok, results} = Connection.mget(conn, ["foo", "bar"]) + assert results == [{:ok, "bar"}, {:error, :not_found}] + + {:ok, results} = Connection.mget(conn, ["foo", "bar"], cas: true) + [{:ok, {"bar", cas}}, {:error, :not_found}] = results + assert is_integer(cas) + + :ok = Connection.set(conn, {"bar", "foo"}) + + {:ok, results} = Connection.mget(conn, ["foo", "bar"]) + assert results == [{:ok, "bar"}, {:ok, "foo"}] + + {:ok, results} = Connection.mget(conn, ["foo", "bar"], cas: true) + [{:ok, {"bar", cas1}}, {:ok, {"foo", cas2}}] = results + assert is_integer(cas1) and is_integer(cas2) + + {:ok, results} = Connection.mset(conn, [ + {"foo", "bar2", cas: cas1+1}, + {"bar", "foo2", cas: cas2}, + ]) + + assert results == [{:error, :exists}, :ok] + + {:ok, results} = Connection.mget(conn, ["foo", "bar"]) + assert results == [{:ok, "bar"}, {:ok, "foo2"}] + + {:ok, results} = Connection.mget(conn, ["foo", "bar"], cas: true) + [{:ok, {"bar", foo_cas}}, {:ok, {"foo2", bar_cas}}] = results + + {:ok, results} = Connection.mset(conn, [ + {"foo", "bar2", cas: foo_cas+1}, + {"bar", "foo3", cas: bar_cas}, + ], cas: true) + [{:error, :exists}, {:ok, bar_cas}] = results + assert is_integer(bar_cas) end end diff --git a/test/support/cluster.ex b/test/support/cluster.ex deleted file mode 100644 index 4cad264..0000000 --- a/test/support/cluster.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Test.Cluster do - use Cream.Cluster, otp_app: :cream -end From c88ce715ac9c987f87505e0773f3940cf9085a7a Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 13 Jun 2021 12:35:50 -0500 Subject: [PATCH 05/28] Progress --- lib/cream/connection.ex | 119 ++++++++++++++++++++++++++++++--------- lib/cream/continuum.ex | 21 ++++--- lib/cream/protocol.ex | 2 +- test/connection_test.exs | 41 +++++++++++--- 4 files changed, 134 insertions(+), 49 deletions(-) diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index d8ea27a..c085fd0 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -3,6 +3,11 @@ defmodule Cream.Connection do require Logger alias Cream.{Protocol} + @type t :: GenServer.server() + @type cas :: non_neg_integer() + @type reason :: atom | binary | term + @type get_result :: {:ok, term} | {:ok, {term, cas}} | {:error, reason} + @defaults [ server: "localhost:11211" ] @@ -21,10 +26,30 @@ defmodule Cream.Connection do Connection.call(conn, {:flush, options}) end - def set(conn, item, options \\ []) do + def set(conn, item, options \\ []) when is_tuple(item) do Connection.call(conn, {:set, item, options}) end + @doc """ + Get a single value. + + ## Options + * `verbose` (boolean, default false) Missing value will return `{:error, :not_found}`. + * `cas` (boolean, default false) Return cas value along with key value. + + ## Examples + + ``` + {:ok, nil} = get(conn, "foo") + + {:error, :not_found} = get(conn, "foo", verbose: true) + + {:ok, "Callie"} = get(conn, "name") + + {:ok, {"Callie", 123}} = get(conn, "name", cas: true) + ``` + """ + @spec get(t, binary, Keyword.t) :: get_result def get(conn, key, options \\ []) do Connection.call(conn, {:get, key, options}) end @@ -33,6 +58,23 @@ defmodule Cream.Connection do Connection.call(conn, {:mset, items, options}) end + @doc """ + Get multiple values. + + ## Options + * `verbose :: boolean \\\\ false` Missing value will return `{:error, :not_found}`. + * `cas :: boolean \\\\ false` Return cas value along with key value. + + ## Examples + ``` + {:ok, [{:ok, nil}, {:ok, "Callie"}]} = mget(conn, ["foo", "name"]) + + {:ok, [{:error, :not_found}, {:ok, "Callie"}]} = mget(conn, ["foo", "name"], verbose: true) + + {:ok, [{:ok, nil}, {:ok, {"Callie", 123}]} = mget(conn, ["foo", "name"], cas: true) + ``` + """ + @spec mget(t, [binary], Keyword.t) :: {:ok, [get_result]} | {:error, reason} def mget(conn, keys, options \\ []) do Connection.call(conn, {:mget, keys, options}) end @@ -120,9 +162,10 @@ defmodule Cream.Connection do def handle_call({:set, item, options}, _from, state) do %{socket: socket} = state - with {:ok, key, value, item_opts} <- parse_item(item), - {:ok, value, flags} <- encode(value, state), - packet = Protocol.set(key, value, flags, item_opts), + item_opts = Keyword.take(options, [:expiry]) + + with {:ok, item} <- normalize_item(item, item_opts, state), + packet = Protocol.set(item), :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), {:ok, packet} <- Protocol.recv_packet(socket), @@ -153,12 +196,23 @@ defmodule Cream.Connection do do case packet.status do :ok -> - {:ok, value} = decode(packet.value, packet.extras.flags, state) - case options[:cas] do - true -> {:reply, {:ok, {value, packet.cas}}, state} - _ -> {:reply, {:ok, value}, state} + with {:ok, value} <- decode(packet.value, packet.extras.flags, state) do + if options[:cas] do + {:reply, {:ok, {value, packet.cas}}, state} + else + {:reply, {:ok, value}, state} + end + else + {:error, reason} -> {:reply, {:error, reason}, state} end - error -> {:reply, {:error, error}, state} + + :not_found -> if options[:verbose] do + {:reply, {:error, :not_found}, state} + else + {:reply, {:ok, nil}, state} + end + + reason -> {:reply, {:error, reason}, state} end else {:error, reason} -> {:disconnect, reason, state} @@ -168,15 +222,17 @@ defmodule Cream.Connection do def handle_call({:mset, items, options}, _from, state) do %{socket: socket} = state - packet_opts = Keyword.take(options, [:expiry]) + item_opts = Keyword.take(options, [:expiry]) + count = Enum.count(items) - with {:ok, packets} <- items_to_packets(items, packet_opts, state), + with {:ok, items} <- normalize_items(items, item_opts, state), + packets = Enum.map(items, &Protocol.set/1), :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packets), - {:ok, packets} <- Protocol.recv_packets(socket, Enum.count(packets)), + {:ok, packets} <- Protocol.recv_packets(socket, count), :ok <- :inet.setopts(socket, active: true) do - results = if options[:cas] do + result = if options[:cas] do Enum.map(packets, fn %{status: :ok, cas: cas} -> {:ok, cas} %{status: reason} -> {:error, reason} @@ -188,7 +244,11 @@ defmodule Cream.Connection do end) end - {:reply, {:ok, results}, state} + if Enum.all?(result, & &1 == :ok) do + {:reply, :ok, state} + else + {:reply, {:ok, result}, state} + end else {:error, reason} -> {:reply, {:error, reason}, state} end @@ -257,29 +317,32 @@ defmodule Cream.Connection do {:disconnect, :tcp_closed, state} end - defp items_to_packets(items, opts, state) do - Enum.reduce_while(items, {:ok, []}, fn item, {:ok, packets} -> - with {:ok, key, value, opts} <- parse_item(item, opts), - {:ok, value, flags} <- encode(value, state) - do - packet = Protocol.set(key, value, flags, opts) - {:cont, {:ok, [packet | packets]}} + defp normalize_items(items, opts, state) do + Enum.reduce_while(items, [], fn item, acc -> + with {:ok, item} <- normalize_item(item, opts, state) do + {:cont, [item | acc]} else - error -> {:halt, error} + {:error, reason} -> {:halt, {:error, reason}} end end) |> case do - {:ok, packets} -> {:ok, Enum.reverse(packets)} - error -> error + {:error, reason} -> {:error, reason} + items -> {:ok, Enum.reverse(items)} end end - defp parse_item(item, options \\ []) do - case item do - {key, value} -> {:ok, key, value, options} - {key, value, opts} -> {:ok, key, value, Keyword.merge(options, opts)} + defp normalize_item(item, opts, state) do + item = case item do + {key, value} -> {:ok, key, value, opts} + {key, value, item_opts} -> {:ok, key, value, Keyword.merge(opts, item_opts)} _ -> {:error, :invalid_item} end + + with {:ok, key, value, opts} <- item, + {:ok, value, flags} <- encode(value, state) + do + {:ok, {key, value, flags, opts}} + end end defp encode(value, %{coder: nil}), do: {:ok, value, 0} diff --git a/lib/cream/continuum.ex b/lib/cream/continuum.ex index 6a33168..d808714 100644 --- a/lib/cream/continuum.ex +++ b/lib/cream/continuum.ex @@ -7,17 +7,16 @@ defmodule Cream.Continuum do total_servers = length(servers) total_weight = length(servers) # TODO implement weights - servers - |> Enum.reduce([], fn server, acc -> - count = entry_count_for(server, 1, total_servers, total_weight) - Enum.reduce 0..count-1, acc, fn i, acc -> - hash = :crypto.hash(:sha, "#{server}:#{i}") |> Base.encode16 - {value, _} = hash |> String.slice(0, 8) |> Integer.parse(16) - [{server, value} | acc] - end - end) - |> Enum.sort_by(fn {_id, value} -> value end) - |> List.to_tuple + Enum.reduce(servers, [], fn server, acc -> + count = entry_count_for(server, 1, total_servers, total_weight) + Enum.reduce 0..count-1, acc, fn i, acc -> + hash = :crypto.hash(:sha, "#{server}:#{i}") |> Base.encode16 + {value, _} = hash |> String.slice(0, 8) |> Integer.parse(16) + [{server, value} | acc] + end + end) + |> Enum.sort_by(fn {_id, value} -> value end) + |> List.to_tuple end def find(continuum, key, attempt \\ 0) diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex index 75444ab..2357d53 100644 --- a/lib/cream/protocol.ex +++ b/lib/cream/protocol.ex @@ -70,7 +70,7 @@ defmodule Cream.Protocol do ] end - def set(key, value, flags, options \\ []) do + def set({key, value, flags, options}) do expiry = Keyword.get(options, :expiry, 0) cas = Keyword.get(options, :cas, 0) diff --git a/test/connection_test.exs b/test/connection_test.exs index a1f00a4..65646f6 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -10,21 +10,44 @@ defmodule ConnectionTest do setup %{conn: conn} do :ok = Connection.flush(conn) + :ok = Connection.set(conn, {"name", "Callie"}) end - test "set/get", %{conn: conn} do - {:error, :not_found} = Connection.get(conn, "foo") - {:error, :not_found} = Connection.get(conn, "foo", cas: true) - + test "set", %{conn: conn} do :ok = Connection.set(conn, {"foo", "bar"}) - + {:ok, cas} = Connection.set(conn, {"foo", "bar"}, cas: true) + {:error, :exists} = Connection.set(conn, {"foo", "bar1", cas: cas+1}) {:ok, "bar"} = Connection.get(conn, "foo") - {:ok, {"bar", cas}} = Connection.get(conn, "foo", cas: true) + :ok = Connection.set(conn, {"foo", "bar1", cas: cas}) + {:ok, "bar1"} = Connection.get(conn, "foo") + end + + test "get", %{conn: conn} do + {:ok, nil} = Connection.get(conn, "foo") + {:error, :not_found} = Connection.get(conn, "foo", verbose: true) + {:ok, "Callie"} = Connection.get(conn, "name") + {:ok, {"Callie", cas}} = Connection.get(conn, "name", cas: true) + assert is_integer(cas) + end + + test "mset", %{conn: conn} do + :ok = Connection.mset(conn, [ + {"foo", "bar"}, + {"name", "Genevieve"} + ]) - {:error, :exists} = Connection.set(conn, {"foo", "baz", cas: cas-1}) {:ok, "bar"} = Connection.get(conn, "foo") - :ok = Connection.set(conn, {"foo", "baz", cas: cas}) - {:ok, "baz"} = Connection.get(conn, "foo") + {:ok, "Genevieve"} = Connection.get(conn, "name") + + {:ok, [ok: foo_cas, ok: name_cas]} = Connection.mset(conn, [ + {"foo", "bar"}, + {"name", "Genevieve"} + ], cas: true) + + {:ok, [{:error, :exists}, :ok]} = Connection.mset(conn, [ + {"foo", "bar", cas: foo_cas+1}, + {"name", "Callie", cas: name_cas} + ]) end test "mset/mget", %{conn: conn} do From 5769ba7f5c7bd7e5058551ee7c7cac4123cea501 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 3 Jul 2021 15:52:48 -0500 Subject: [PATCH 06/28] Simplify Connection API, coders, docs --- lib/cream/client.ex | 64 +++++++ lib/cream/cluster.ex | 5 + lib/cream/coder.ex | 40 ++++ lib/cream/coder/gzip.ex | 26 +++ lib/cream/coder/jason.ex | 22 +++ lib/cream/connection.ex | 397 +++++++++++++++++++++++---------------- lib/cream/protocol.ex | 11 +- lib/cream/utils.ex | 16 ++ mix.exs | 3 + mix.lock | 8 + test/coder_test.exs | 57 ++++++ test/connection_test.exs | 67 +------ test/low_level_test.exs | 180 ------------------ 13 files changed, 486 insertions(+), 410 deletions(-) create mode 100644 lib/cream/client.ex create mode 100644 lib/cream/coder.ex create mode 100644 lib/cream/coder/gzip.ex create mode 100644 lib/cream/coder/jason.ex create mode 100644 lib/cream/utils.ex create mode 100644 test/coder_test.exs delete mode 100644 test/low_level_test.exs diff --git a/lib/cream/client.ex b/lib/cream/client.ex new file mode 100644 index 0000000..c637718 --- /dev/null +++ b/lib/cream/client.ex @@ -0,0 +1,64 @@ +defmodule Cream.Client do + @behaviour NimblePool + + @defaults [ + pool_size: 5, + lazy: true, + servers: ["localhost:11211"] + ] + def defaults, do: @defaults + + def config do + Keyword.merge( + @defaults, + Application.get_application(__MODULE__) + |> Application.get_env(__MODULE__, []) + ) + end + + @impl NimblePool + def init_pool(config) do + config = Map.new(config) + |> Map.merge(%{ + continuum: Cream.Continuum.new(config[:servers]) + }) + + {:ok, config} + end + + @impl NimblePool + def init_worker(config) do + worker = Map.new(config[:servers], fn server -> + {:ok, conn} = Cream.Connection.start_link(server: server) + {server, conn} + end) + + {:ok, worker, config} + end + + @impl NimblePool + def handle_checkout(:checkout, _from, worker, config) do + client = Map.put(config, :connections, worker) + {:ok, client, worker, config} + end + + def child_spec(config \\ []) do + config = Keyword.merge(config(), config) + + Keyword.take(config, [:pool_size, :lazy]) + |> Keyword.put(:worker, {__MODULE__, config}) + |> NimblePool.child_spec() + end + + def start_link(config \\ []) do + %{start: {m, f, a}} = child_spec(config) + apply(m, f, a) + end + + def checkout(pool, f) do + NimblePool.checkout!(pool, :checkout, fn _, client -> + {f.(client), client} + end) + end + +end diff --git a/lib/cream/cluster.ex b/lib/cream/cluster.ex index 151ddc8..f46ae72 100644 --- a/lib/cream/cluster.ex +++ b/lib/cream/cluster.ex @@ -1,3 +1,8 @@ defmodule Cream.Cluster do + defstruct [:continuum, :servers] + + def set(item) do + + end end diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex new file mode 100644 index 0000000..e422225 --- /dev/null +++ b/lib/cream/coder.ex @@ -0,0 +1,40 @@ +defmodule Cream.Coder do + + @type value :: term + @type flags :: integer() + @type reason :: term + + @callback encode(value, flags) :: {:ok, value, flags} | {:error, reason} + @callback decode(value, flags) :: {:ok, value} | {:error, reason} + + @doc false + def apply_encode(coder, value, flags) when is_atom(coder) do + coder.encode(value, flags) + end + + @doc false + def apply_encode(coders, value, flags) when is_list(coders) do + Enum.reduce_while(coders, {:ok, value, flags}, fn coder, {:ok, value, flags} -> + case apply_encode(coder, value, flags) do + {:ok, value, flags} -> {:cont, {:ok, value, flags}} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + @doc false + def apply_decode(coder, value, flags) when is_atom(coder) do + coder.decode(value, flags) + end + + @doc false + def apply_decode(coders, value, flags) when is_list(coders) do + Enum.reduce_while(coders, {:ok, value}, fn coder, {:ok, value} -> + case apply_decode(coder, value, flags) do + {:ok, value} -> {:cont, {:ok, value}} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + +end diff --git a/lib/cream/coder/gzip.ex b/lib/cream/coder/gzip.ex new file mode 100644 index 0000000..7426a63 --- /dev/null +++ b/lib/cream/coder/gzip.ex @@ -0,0 +1,26 @@ +defmodule Cream.Coder.Gzip do + use Bitwise + + @behaviour Cream.Coder + + def encode(value, flags) when is_binary(value) do + {:ok, :zlib.gzip(value), flags ||| 0b10} + end + + def encode(_value, _flags) do + {:error, "cannot gzip non-binary values"} + end + + def decode(value, flags) when (flags &&& 0b10) == 0b10 do + if is_binary(value) do + {:ok, :zlib.gunzip(value)} + else + {:error, "cannot gunzip non-binary values"} + end + end + + def decode(value, _flags) do + {:ok, value} + end + +end diff --git a/lib/cream/coder/jason.ex b/lib/cream/coder/jason.ex new file mode 100644 index 0000000..22899de --- /dev/null +++ b/lib/cream/coder/jason.ex @@ -0,0 +1,22 @@ +defmodule Cream.Coder.Jason do + use Bitwise + + @behaviour Cream.Coder + + def encode(value, flags) do + with {:ok, json} <- Jason.encode(value) do + {:ok, json, flags ||| 0b01} + end + end + + def decode(value, flags) when (flags &&& 0b01) == 0b01 do + with {:ok, json} <- Jason.decode(value) do + {:ok, json} + end + end + + def decode(value, _flags) do + {:ok, value} + end + +end diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index c085fd0..48b669f 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -1,86 +1,277 @@ defmodule Cream.Connection do + @moduledoc """ + Basic connection to a single memcached server. + + ## Global configuration + + You can globally configure _all connections_ via `Config`. + ``` + import Config + + config :cream, Cream.Connection, server: "foo.bar.com:11211" + ``` + + Now every single connection will use `"foo.bar.com:11211"` for `:server` + unless overwritten by an argument passed to `start_link/1` or `child_spec/1`. + + ## Reconnecting + + This uses the awesome `Connection` behaviour and also "active" sockets, which + means a few things... + 1. `start_link/1` will typically always succeed. + 1. Disconnections are detected immediately. + 1. Reconnections are retried indefinitely. + + While a connection is in a disconnected state, any operations on the + connection will result in `{:error, :not_connected}`. + + ## Coders + + Coders serialize and deserialize item values. They are also responsible for + setting flags on items. + + Let's make a simple JSON coder. + + ``` + defmodule JsonCoder do + @behaviour Cream.Coder + use Bitwise + + def encode(value, flags) when is_map(value) do + value = Jason.encode!(value) + flags = flags &&& 0x1 + {:ok, value, flags} + end + + def encode(value, flags) do + flags = flags &&& 0x0 + {:ok, value, flags} + end + + def decode(value, flags) when flags &&& 0x1 do + {:ok, Jason.decode!(value)} + end + + def decode(value, flags) do + {:ok, value} + end + end + ``` + """ + use Connection require Logger alias Cream.{Protocol} + @typedoc """ + A connection. + """ @type t :: GenServer.server() - @type cas :: non_neg_integer() + + @typedoc """ + Item used with `set/3`. + + An item is a key/value tuple or a key/value/opts tuple. + + ``` + {"foo", "bar"} + {"foo", "bar", ttl: 60} + {"foo", "bar", cas: 123} + {"foo", "bar", ttl: 60, cas: 123} + ``` + """ + @type item :: {binary, term} | {binary, term, Keyword.t} + + @typedoc """ + An error reason. + """ @type reason :: atom | binary | term - @type get_result :: {:ok, term} | {:ok, {term, cas}} | {:error, reason} + + @typedoc """ + Check and set value. + """ + @type cas :: non_neg_integer() @defaults [ - server: "localhost:11211" + server: "localhost:11211", + coder: nil, ] - def start_link(config \\ []) do - mix_config = Application.get_application(__MODULE__) + @doc """ + Default config. + + ``` + #{inspect @defaults, pretty: true, width: 0} + ``` + """ + def defaults, do: @defaults + + @doc """ + `Config` merged with `defaults/0`. + + ``` + import Config + config :cream, Cream.Connection, coder: FooCoder + + iex(1)> Cream.Connection.config() + #{inspect Keyword.merge(@defaults, coder: FooCoder), pretty: true, width: 0} + ``` + """ + def config do + config = Application.get_application(__MODULE__) |> Application.get_env(__MODULE__, []) - config = Keyword.merge(@defaults, mix_config) - |> Keyword.merge(config) + Keyword.merge(@defaults, config) + end + + @doc """ + Start connection. + + The given `config` will be merged over `config/0`. + + See `defaults/0` for config options. + + Note that this will typically _always_ return `{:ok conn}`, even if the actual + TCP connection failed. If that is the case, subsequent uses of the connection + will return `{:error, :not_connected}`. + + ``` + {:ok, conn} = #{inspect __MODULE__}.start_link() + {:ok, conn} = #{inspect __MODULE__}.start_link(server: "memcache.foo.com:11211") + ``` + """ + @spec start_link(Keyword.t) :: {:ok, t} | {:error, reason} + def start_link(config \\ []) do + config = Keyword.merge(config(), config) Connection.start_link(__MODULE__, config) end - def flush(conn, options \\ []) do - Connection.call(conn, {:flush, options}) - end + @doc """ + Clear all keys. + + ``` + # Flush immediately + iex(1)> #{inspect __MODULE__}.flush(conn) + :ok - def set(conn, item, options \\ []) when is_tuple(item) do - Connection.call(conn, {:set, item, options}) + # Flush after 60 seconds + iex(1)> #{inspect __MODULE__}.flush(conn, ttl: 60) + :ok + ``` + """ + @spec flush(t, Keyword.t) :: :ok | {:error, reason} + def flush(conn, opts \\ []) do + Connection.call(conn, {:flush, opts}) end @doc """ - Get a single value. + Set a single item. ## Options - * `verbose` (boolean, default false) Missing value will return `{:error, :not_found}`. - * `cas` (boolean, default false) Return cas value along with key value. + + * `ttl` (`integer`) - Apply time to live (expiry) to the item. Default `nil`. + * `return_cas` (`boolean`) - Return cas value in result. Default `false`. + + If `ttl` is set explicitly on the item, that will take precedence over + the `ttl` specified in `opts`. ## Examples ``` - {:ok, nil} = get(conn, "foo") + # Basic set + iex(1)> set(conn, {"foo", "bar"}) + :ok + + # Use ttl on item. + iex(1)> set(conn, {"foo", "bar", ttl: 60}) + :ok - {:error, :not_found} = get(conn, "foo", verbose: true) + # Use ttl from opts. + iex(1)> set(conn, {"foo", "bar"}, ttl: 60) + :ok - {:ok, "Callie"} = get(conn, "name") + # Return cas value. + iex(1)> set(conn, {"foo", "bar"}, return_cas: true) + {:ok, 123} - {:ok, {"Callie", 123}} = get(conn, "name", cas: true) + # Set using bad cas value results in error. + iex(1)> set(conn, {"foo", "bar", cas: 124}) + {:error, :exists} + + # Set using cas value and return new cas value. + iex(1)> set(conn, {"foo", "bar", cas: 123}, return_cas: true) + {:ok, 124} + + # Set using cas. + iex(1)> set(conn, {"foo", "bar", cas: 124}) + :ok ``` """ - @spec get(t, binary, Keyword.t) :: get_result - def get(conn, key, options \\ []) do - Connection.call(conn, {:get, key, options}) - end - - def mset(conn, items, options \\ []) do - Connection.call(conn, {:mset, items, options}) + @spec set(t, item, Keyword.t) :: :ok | {:ok, cas} | {:error, reason} + def set(conn, item, opts \\ []) do + item = Cream.Utils.normalize_item(item) + Connection.call(conn, {:set, item, opts}) end @doc """ - Get multiple values. + Get a single item. ## Options - * `verbose :: boolean \\\\ false` Missing value will return `{:error, :not_found}`. - * `cas :: boolean \\\\ false` Return cas value along with key value. + + * `verbose` (boolean, default false) Missing value will return `{:error, :not_found}`. + * `cas` (boolean, default false) Return cas value in result. ## Examples + ``` - {:ok, [{:ok, nil}, {:ok, "Callie"}]} = mget(conn, ["foo", "name"]) + iex(1)> get(conn, "foo") + {:ok, nil} + + iex(1)> get(conn, "foo", verbose: true) + {:error, :not_found} - {:ok, [{:error, :not_found}, {:ok, "Callie"}]} = mget(conn, ["foo", "name"], verbose: true) + iex(1)> get(conn, "name") + {:ok, "Callie"} - {:ok, [{:ok, nil}, {:ok, {"Callie", 123}]} = mget(conn, ["foo", "name"], cas: true) + iex(1)> get(conn, "name", return_cas: true) + {:ok, "Callie", 123} ``` """ - @spec mget(t, [binary], Keyword.t) :: {:ok, [get_result]} | {:error, reason} - def mget(conn, keys, options \\ []) do - Connection.call(conn, {:mget, keys, options}) + @spec get(t, binary, Keyword.t) :: {:ok, term} | {:ok, term, cas} | {:error, reason} + def get(conn, key, options \\ []) do + case Connection.call(conn, {:get, key, options}) do + {:error, :not_found} = result -> if options[:verbose] do + result + else + {:ok, nil} + end + + result -> result + end end + @doc """ + Send a noop command to server. + + You can use this to see if you are connected to the server. + ``` + iex(1)> noop(conn) + :ok + + iex(1)> noop(conn) + {:error, :not_connected} + ``` + """ + @spec noop(t) :: :ok | {:error, reason} def noop(conn) do - Connection.call(conn, :noop) + case Connection.call(conn, :noop) do + {:ok, packet} -> case packet do + %{status: :ok} -> :ok + %{status: reason} -> {:error, reason} + end + error -> error + end end def init(config) do @@ -159,20 +350,20 @@ defmodule Cream.Connection do end end - def handle_call({:set, item, options}, _from, state) do + def handle_call({:set, item, opts}, _from, state) do %{socket: socket} = state - item_opts = Keyword.take(options, [:expiry]) + {key, value, ttl, cas} = item - with {:ok, item} <- normalize_item(item, item_opts, state), - packet = Protocol.set(item), + with {:ok, value, flags} <- encode(value, state), + packet = Protocol.set({key, value, ttl, cas, flags}), :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do case packet.status do - :ok -> if options[:cas] do + :ok -> if opts[:return_cas] do {:reply, {:ok, packet.cas}, state} else {:reply, :ok, state} @@ -184,10 +375,10 @@ defmodule Cream.Connection do end end - def handle_call({:get, key, options}, _from, state) do + def handle_call({:get, key, opts}, _from, state) do %{socket: socket} = state - packet = Protocol.get(key, options) + packet = Protocol.get(key) with :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), @@ -197,8 +388,8 @@ defmodule Cream.Connection do case packet.status do :ok -> with {:ok, value} <- decode(packet.value, packet.extras.flags, state) do - if options[:cas] do - {:reply, {:ok, {value, packet.cas}}, state} + if opts[:return_cas] do + {:reply, {:ok, value, packet.cas}, state} else {:reply, {:ok, value}, state} end @@ -206,12 +397,6 @@ defmodule Cream.Connection do {:error, reason} -> {:reply, {:error, reason}, state} end - :not_found -> if options[:verbose] do - {:reply, {:error, :not_found}, state} - else - {:reply, {:ok, nil}, state} - end - reason -> {:reply, {:error, reason}, state} end else @@ -219,86 +404,6 @@ defmodule Cream.Connection do end end - def handle_call({:mset, items, options}, _from, state) do - %{socket: socket} = state - - item_opts = Keyword.take(options, [:expiry]) - count = Enum.count(items) - - with {:ok, items} <- normalize_items(items, item_opts, state), - packets = Enum.map(items, &Protocol.set/1), - :ok <- :inet.setopts(socket, active: false), - :ok <- :gen_tcp.send(socket, packets), - {:ok, packets} <- Protocol.recv_packets(socket, count), - :ok <- :inet.setopts(socket, active: true) - do - result = if options[:cas] do - Enum.map(packets, fn - %{status: :ok, cas: cas} -> {:ok, cas} - %{status: reason} -> {:error, reason} - end) - else - Enum.map(packets, fn - %{status: :ok} -> :ok - %{status: reason} -> {:error, reason} - end) - end - - if Enum.all?(result, & &1 == :ok) do - {:reply, :ok, state} - else - {:reply, {:ok, result}, state} - end - else - {:error, reason} -> {:reply, {:error, reason}, state} - end - end - - def handle_call({:mget, keys, options}, _from, state) do - %{socket: socket} = state - - packets = Enum.map(keys, fn - {key, _opts} -> Protocol.getkq(to_string(key)) - key -> Protocol.getkq(to_string(key)) - end) - - packets = [packets, Protocol.noop()] - - with :ok <- :inet.setopts(socket, active: false), - :ok <- :gen_tcp.send(socket, packets), - {:ok, packets} <- Protocol.recv_packets(socket, :noop), - :ok <- :inet.setopts(socket, active: true) - do - values_by_key = Enum.reduce_while(packets, %{}, fn packet, acc -> - case packet.opcode do - :noop -> {:halt, acc} - _ -> {:cont, Map.put(acc, packet.key, {packet.value, packet.cas})} - end - end) - - cas = Keyword.get(options, :cas, false) - - responses = Enum.map(keys, fn key -> - {key, opts} = case key do - {key, opts} -> {key, opts} - key -> {key, []} - end - - cas = opts[:cas] || cas - - case {values_by_key[key], cas} do - {nil, _} -> {:error, :not_found} - {{value, _cas}, false} -> {:ok, value} - {value_cas, true} -> {:ok, value_cas} - end - end) - - {:reply, {:ok, responses}, state} - else - {:error, reason} -> {:disconnect, reason, state} - end - end - def handle_call(:noop, _from, state) do %{socket: socket} = state @@ -317,37 +422,9 @@ defmodule Cream.Connection do {:disconnect, :tcp_closed, state} end - defp normalize_items(items, opts, state) do - Enum.reduce_while(items, [], fn item, acc -> - with {:ok, item} <- normalize_item(item, opts, state) do - {:cont, [item | acc]} - else - {:error, reason} -> {:halt, {:error, reason}} - end - end) - |> case do - {:error, reason} -> {:error, reason} - items -> {:ok, Enum.reverse(items)} - end - end - - defp normalize_item(item, opts, state) do - item = case item do - {key, value} -> {:ok, key, value, opts} - {key, value, item_opts} -> {:ok, key, value, Keyword.merge(opts, item_opts)} - _ -> {:error, :invalid_item} - end - - with {:ok, key, value, opts} <- item, - {:ok, value, flags} <- encode(value, state) - do - {:ok, {key, value, flags, opts}} - end - end - defp encode(value, %{coder: nil}), do: {:ok, value, 0} defp encode(value, %{coder: coder}) do - coder.encode(value) + {coder.encode(value), 0} end defp decode(value, _flags, %{coder: nil}), do: {:ok, value} diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex index 2357d53..8eccf45 100644 --- a/lib/cream/protocol.ex +++ b/lib/cream/protocol.ex @@ -54,7 +54,7 @@ defmodule Cream.Protocol do end def flush(options \\ []) do - expiry = Keyword.get(options, :expiry, 0) + expiry = Keyword.get(options, :ttl, 0) [ <<@request::bytes(1)>>, @@ -70,10 +70,7 @@ defmodule Cream.Protocol do ] end - def set({key, value, flags, options}) do - expiry = Keyword.get(options, :expiry, 0) - cas = Keyword.get(options, :cas, 0) - + def set({key, value, ttl, cas, flags}) do key_size = byte_size(key) value_size = byte_size(value) body_size = key_size + value_size + 8 @@ -89,7 +86,7 @@ defmodule Cream.Protocol do <<0x00::bytes(4)>>, <>, <>, - <>, + <>, key, value ] @@ -120,7 +117,7 @@ defmodule Cream.Protocol do ] end - def get(key, _options \\ []) do + def get(key) do key_size = byte_size(key) [ diff --git a/lib/cream/utils.ex b/lib/cream/utils.ex new file mode 100644 index 0000000..90011e7 --- /dev/null +++ b/lib/cream/utils.ex @@ -0,0 +1,16 @@ +defmodule Cream.Utils do + + def normalize_item(item, opts \\ []) + + def normalize_item({key, value}, opts) do + ttl = Keyword.get(opts, :ttl, 0) + {key, value, ttl, 0} + end + + def normalize_item({key, value, item_opts}, opts) do + ttl = item_opts[:ttl] || opts[:ttl] || 0 + cas = Keyword.get(item_opts, :cas, 0) + {key, value, ttl, cas} + end + +end diff --git a/mix.exs b/mix.exs index 8aace18..89ce9bb 100644 --- a/mix.exs +++ b/mix.exs @@ -60,6 +60,9 @@ defmodule Cream.Mixfile do [ {:telemetry, "~> 0.4.2"}, {:connection, "~> 1.0"}, + {:nimble_pool, "~> 0.0"}, + {:jason, "~> 1.0", only: [:dev, :test]}, + {:ex_doc, "~> 0.0", only: :dev}, ] end end diff --git a/mix.lock b/mix.lock index 1ea0e43..5be89b7 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,12 @@ %{ "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, + "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"}, + "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, + "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "nimble_pool": {:hex, :nimble_pool, "0.2.4", "1db8e9f8a53d967d595e0b32a17030cdb6c0dc4a451b8ac787bf601d3f7704c3", [:mix], [], "hexpm", "367e8071e137b787764e6a9992ccb57b276dc2282535f767a07d881951ebeac6"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, } diff --git a/test/coder_test.exs b/test/coder_test.exs new file mode 100644 index 0000000..836b213 --- /dev/null +++ b/test/coder_test.exs @@ -0,0 +1,57 @@ +defmodule Coder.Test do + use ExUnit.Case + + alias Cream.Coder + + test "jason" do + {:ok, json, 0b1} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0) + {:ok, %{"foo" => "bar"}} = Jason.decode(json) + + {:ok, json, 0b101} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0b100) + {:ok, %{"foo" => "bar"}} = Jason.decode(json) + + {:error, _reason} = Coder.apply_encode(Coder.Jason, {"foo", "bar"}, 0b100) + + {:ok, json, flags} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0) + {:ok, %{"foo" => "bar"}} = Coder.apply_decode(Coder.Jason, json, flags) + + {:ok, json, flags} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0b100) + {:ok, %{"foo" => "bar"}} = Coder.apply_decode(Coder.Jason, json, flags) + end + + test "gzip" do + {:ok, zipped, 0b10} = Coder.apply_encode(Coder.Gzip, "foobar", 0) + "foobar" = :zlib.gunzip(zipped) + + {:ok, zipped, 0b110} = Coder.apply_encode(Coder.Gzip, "foobar", 0b100) + "foobar" = :zlib.gunzip(zipped) + + {:error, _reason} = Coder.apply_encode(Coder.Gzip, :foobar, 0) + + {:ok, zipped, flags} = Coder.apply_encode(Coder.Gzip, "foobar", 0) + {:ok, "foobar"} = Coder.apply_decode(Coder.Gzip, zipped, flags) + + {:ok, zipped, flags} = Coder.apply_encode(Coder.Gzip, "foobar", 0b100) + {:ok, "foobar"} = Coder.apply_decode(Coder.Gzip, zipped, flags) + end + + test "jason + gzip" do + encoders = [Coder.Jason, Coder.Gzip] + decoders = [Coder.Gzip, Coder.Jason] + + term = %{"foo" => "bar"} + + {:ok, data, 0b11} = Coder.apply_encode(encoders, term, 0) + ^term = data |> :zlib.gunzip() |> Jason.decode!() + + {:ok, data, 0b111} = Coder.apply_encode(encoders, term, 0b100) + ^term = data |> :zlib.gunzip() |> Jason.decode!() + + {:ok, data, 0b11} = Coder.apply_encode(encoders, term, 0) + {:ok, ^term} = Coder.apply_decode(decoders, data, 0b11) + + {:ok, data, flags} = Coder.apply_encode(encoders, term, 0b100) + {:ok, ^term} = Coder.apply_decode(decoders, data, flags) + end + +end diff --git a/test/connection_test.exs b/test/connection_test.exs index 65646f6..34a6a54 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -10,12 +10,11 @@ defmodule ConnectionTest do setup %{conn: conn} do :ok = Connection.flush(conn) - :ok = Connection.set(conn, {"name", "Callie"}) end test "set", %{conn: conn} do :ok = Connection.set(conn, {"foo", "bar"}) - {:ok, cas} = Connection.set(conn, {"foo", "bar"}, cas: true) + {:ok, cas} = Connection.set(conn, {"foo", "bar"}, return_cas: true) {:error, :exists} = Connection.set(conn, {"foo", "bar1", cas: cas+1}) {:ok, "bar"} = Connection.get(conn, "foo") :ok = Connection.set(conn, {"foo", "bar1", cas: cas}) @@ -23,71 +22,13 @@ defmodule ConnectionTest do end test "get", %{conn: conn} do + :ok = Connection.set(conn, {"name", "Callie"}) + {:ok, nil} = Connection.get(conn, "foo") {:error, :not_found} = Connection.get(conn, "foo", verbose: true) {:ok, "Callie"} = Connection.get(conn, "name") - {:ok, {"Callie", cas}} = Connection.get(conn, "name", cas: true) + {:ok, "Callie", cas} = Connection.get(conn, "name", return_cas: true) assert is_integer(cas) end - test "mset", %{conn: conn} do - :ok = Connection.mset(conn, [ - {"foo", "bar"}, - {"name", "Genevieve"} - ]) - - {:ok, "bar"} = Connection.get(conn, "foo") - {:ok, "Genevieve"} = Connection.get(conn, "name") - - {:ok, [ok: foo_cas, ok: name_cas]} = Connection.mset(conn, [ - {"foo", "bar"}, - {"name", "Genevieve"} - ], cas: true) - - {:ok, [{:error, :exists}, :ok]} = Connection.mset(conn, [ - {"foo", "bar", cas: foo_cas+1}, - {"name", "Callie", cas: name_cas} - ]) - end - - test "mset/mget", %{conn: conn} do - :ok = Connection.set(conn, {"foo", "bar"}) - - {:ok, results} = Connection.mget(conn, ["foo", "bar"]) - assert results == [{:ok, "bar"}, {:error, :not_found}] - - {:ok, results} = Connection.mget(conn, ["foo", "bar"], cas: true) - [{:ok, {"bar", cas}}, {:error, :not_found}] = results - assert is_integer(cas) - - :ok = Connection.set(conn, {"bar", "foo"}) - - {:ok, results} = Connection.mget(conn, ["foo", "bar"]) - assert results == [{:ok, "bar"}, {:ok, "foo"}] - - {:ok, results} = Connection.mget(conn, ["foo", "bar"], cas: true) - [{:ok, {"bar", cas1}}, {:ok, {"foo", cas2}}] = results - assert is_integer(cas1) and is_integer(cas2) - - {:ok, results} = Connection.mset(conn, [ - {"foo", "bar2", cas: cas1+1}, - {"bar", "foo2", cas: cas2}, - ]) - - assert results == [{:error, :exists}, :ok] - - {:ok, results} = Connection.mget(conn, ["foo", "bar"]) - assert results == [{:ok, "bar"}, {:ok, "foo2"}] - - {:ok, results} = Connection.mget(conn, ["foo", "bar"], cas: true) - [{:ok, {"bar", foo_cas}}, {:ok, {"foo2", bar_cas}}] = results - - {:ok, results} = Connection.mset(conn, [ - {"foo", "bar2", cas: foo_cas+1}, - {"bar", "foo3", cas: bar_cas}, - ], cas: true) - [{:error, :exists}, {:ok, bar_cas}] = results - assert is_integer(bar_cas) - end - end diff --git a/test/low_level_test.exs b/test/low_level_test.exs deleted file mode 100644 index 560ba09..0000000 --- a/test/low_level_test.exs +++ /dev/null @@ -1,180 +0,0 @@ -defmodule LowLevelTest do - use ExUnit.Case - - alias Cream.{Connection, Packet} - - setup_all do - {:ok, conn} = Connection.start_link() - [conn: conn] - end - - test "noop", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:noop)]) - {:ok, [%{opcode: :noop}]} = Connection.recv_packets(conn, 1) - end - - test "flush", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:flush)]) - {:ok, [%{opcode: :flush}]} = Connection.recv_packets(conn, 1) - end - - describe "set, add, replace" do - - setup :flush - - test "set", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:set, key: "foo", value: "bar")]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :set - assert packet.status == :ok - end - - test "setq", %{conn: conn} do - :ok = Connection.send_packets(conn, [ - Packet.new(:setq, key: "foo", value: "bar"), - Packet.new(:set, key: "bar", value: "foo"), - ]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :set - assert packet.status == :ok - end - - end - - describe "retrieving (misses)" do - - setup :flush - - test "get", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:get, key: "foo")]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :get - assert packet.status == :not_found - assert packet.key == "" - assert packet.value == "Not found" - end - - test "getk", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:getk, key: "foo")]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :getk - assert packet.status == :not_found - assert packet.key == "foo" - assert packet.value == "" - end - - test "getq", %{conn: conn} do - :ok = Connection.send_packets(conn, [ - Packet.new(:getq, key: "foo"), - Packet.new(:getk, key: "bar") - ]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :getk - assert packet.status == :not_found - assert packet.key == "bar" - assert packet.value == "" - end - - test "getqk", %{conn: conn} do - :ok = Connection.send_packets(conn, [ - Packet.new(:getkq, key: "foo"), - Packet.new(:getk, key: "bar") - ]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :getk - assert packet.status == :not_found - assert packet.key == "bar" - assert packet.value == "" - end - - end - - describe "retrieving (hits)" do - - setup :flush - - setup %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:set, key: "foo", value: "bar")]) - {:ok, [%{opcode: :set}]} = Connection.recv_packets(conn, 1) - [conn: conn] - end - - test "get", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:get, key: "foo")]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :get - assert packet.status == :ok - assert packet.key == "" - assert packet.value == "bar" - end - - test "getq", %{conn: conn} do - :ok = Connection.send_packets(conn, [ - Packet.new(:getq, key: "baz"), - Packet.new(:getq, key: "foo") - ]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :getq - assert packet.status == :ok - assert packet.key == "" - assert packet.value == "bar" - end - - test "getk", %{conn: conn} do - :ok = Connection.send_packets(conn, [Packet.new(:getk, key: "foo")]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :getk - assert packet.status == :ok - assert packet.key == "foo" - assert packet.value == "bar" - end - - test "getkq", %{conn: conn} do - :ok = Connection.send_packets(conn, [ - Packet.new(:getkq, key: "baz"), - Packet.new(:getkq, key: "foo") - ]) - {:ok, [packet]} = Connection.recv_packets(conn, 1) - - assert packet.opcode == :getkq - assert packet.status == :ok - assert packet.key == "foo" - assert packet.value == "bar" - end - - end - - describe "extras (flags)" do - setup :flush - - test "set", %{conn: conn} do - :ok = Connection.send_packets(conn, [ - Packet.new(:set, key: "foo", value: "bar", flags: 123), - Packet.new(:get, key: "foo") - ]) - {:ok, [_, packet]} = Connection.recv_packets(conn, 2) - - assert packet.opcode == :get - assert packet.status == :ok - assert packet.value == "bar" - assert packet.extra.flags == 123 - end - - end - - def flush(%{conn: conn}) do - :ok = Connection.send_packets(conn, [Packet.new(:flush)]) - {:ok, [%{opcode: :flush}]} = Connection.recv_packets(conn, 1) - :ok - end - -end From b0649db7ce0e884a4c0dd125273a1a397890e9ba Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Mon, 5 Jul 2021 15:02:21 -0500 Subject: [PATCH 07/28] Polish up coders, tests, and docs --- lib/cream/coder.ex | 104 ++++++++++++++++++++++++++++++++++--- lib/cream/connection.ex | 72 ++++++++++--------------- lib/cream/protocol.ex | 1 + lib/cream/utils.ex | 1 + test/coder_test.exs | 40 +++++++------- test/connection_test.exs | 31 ++++++++++- test/support/coder/json.ex | 26 ++++++++++ 7 files changed, 203 insertions(+), 72 deletions(-) create mode 100644 test/support/coder/json.ex diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index e422225..001adfc 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -1,21 +1,113 @@ defmodule Cream.Coder do + @moduledoc """ + Value serialization. + + Coders serialize and deserialize values when writing and reading to memcached. + + ## JSON example + + Let's make a simple JSON coder... + + ``` + defmodule JsonCoder do + use Bitwise + + @behaviour Cream.Coder + + # Do not encode binary values. + def encode(value, flags) when is_binary(value) do + {:ok, value, flags &&& 0b0} + end + + # Do encode everything else. + def encode(value, flags) do + with {:ok, json} <- Jason.encode(value) do + {:ok, json, flags ||| 0b1} + end + end + + # Flags indicates the value isn't encoded. + def decode(value, flags) when (flags ||| 0b1) == 0 do + {:ok, value} + end + + # Flags indicate we have an encoded value. + def decode(json, flags) when (flags ||| 0b1) == 0b1 do + with {:ok, value} <- Jason.decode(json) do + {:ok, value} + end + end + + end + ``` + + Notice we set a bit (`0b1`) on flags to indicate the value is serialized. When + the value is not serialized, we unset the bit. + + Now let's see the coder in action. We use two connections, one that uses the + coder and one that doesn't. + ``` + {:ok, json_conn} = Cream.Connection.start_link(coder: JsonCoder) + {:ok, raw_conn} = Cream.Connection.start_link() + + # JsonCoder serializes maps, but leaves strings alone. + + iex(1)> Cream.Connection.set(json_conn, "foo", %{"hello" => "world"}) + :ok + + iex(1)> Cream.Connection.get(json_conn, "foo") + {:ok, %{"hello" => "world"}} + + iex(1)> Cream.Connection.set(json_conn, "bar", "hello world") + :ok + + iex(1)> Cream.Connection.get(json_conn, "bar") + {:ok, "hello world"} + + # We can verify this by using the connection that doesn't use the coder. + + iex(1)> Cream.Connection.get(raw_conn, "foo") + {:ok, "{\\"hello\\":\\"world\\"}"} + + iex(1)> Cream.Connection.get(raw_conn, "bar") + {:ok, "hello world"} + ``` + + ## Multiple coders + + You can specify a chain of coders. + + ``` + {:ok, :conn} = Cream.Connection.start_link(coder: [JsonCoder, GzipCoder]) + ``` + + Encoding is done in the order specified and decoding is done in the reverse + order. + """ @type value :: term @type flags :: integer() @type reason :: term + @doc """ + Encode a value and set flags. + """ @callback encode(value, flags) :: {:ok, value, flags} | {:error, reason} + + @doc """ + Decode a value based on flags. + """ @callback decode(value, flags) :: {:ok, value} | {:error, reason} @doc false - def apply_encode(coder, value, flags) when is_atom(coder) do + def encode(coder, value, flags) when is_atom(coder) do coder.encode(value, flags) end @doc false - def apply_encode(coders, value, flags) when is_list(coders) do + def encode(coders, value, flags) when is_list(coders) do Enum.reduce_while(coders, {:ok, value, flags}, fn coder, {:ok, value, flags} -> - case apply_encode(coder, value, flags) do + case encode(coder, value, flags) do {:ok, value, flags} -> {:cont, {:ok, value, flags}} {:error, reason} -> {:halt, {:error, reason}} end @@ -23,14 +115,14 @@ defmodule Cream.Coder do end @doc false - def apply_decode(coder, value, flags) when is_atom(coder) do + def decode(coder, value, flags) when is_atom(coder) do coder.decode(value, flags) end @doc false - def apply_decode(coders, value, flags) when is_list(coders) do + def decode(coders, value, flags) when is_list(coders) do Enum.reduce_while(coders, {:ok, value}, fn coder, {:ok, value} -> - case apply_decode(coder, value, flags) do + case decode(coder, value, flags) do {:ok, value} -> {:cont, {:ok, value}} {:error, reason} -> {:halt, {:error, reason}} end diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 48b669f..420d804 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -16,52 +16,26 @@ defmodule Cream.Connection do ## Reconnecting - This uses the awesome `Connection` behaviour and also "active" sockets, which - means a few things... + `Cream.Connection` uses the awesome `Connection` behaviour and also "active" + sockets, which means a few things... 1. `start_link/1` will typically always succeed. 1. Disconnections are detected immediately. - 1. Reconnections are retried indefinitely. + 1. (Re)connections are retried indefinitely. While a connection is in a disconnected state, any operations on the connection will result in `{:error, :not_connected}`. - ## Coders + ## Value serialization - Coders serialize and deserialize item values. They are also responsible for - setting flags on items. + Coders serialize and deserialize values. They are also responsible for setting + flags on values. - Let's make a simple JSON coder. - - ``` - defmodule JsonCoder do - @behaviour Cream.Coder - use Bitwise - - def encode(value, flags) when is_map(value) do - value = Jason.encode!(value) - flags = flags &&& 0x1 - {:ok, value, flags} - end - - def encode(value, flags) do - flags = flags &&& 0x0 - {:ok, value, flags} - end - - def decode(value, flags) when flags &&& 0x1 do - {:ok, Jason.decode!(value)} - end - - def decode(value, flags) do - {:ok, value} - end - end - ``` + See `Cream.Coder` for more info. """ use Connection require Logger - alias Cream.{Protocol} + alias Cream.{Protocol, Coder} @typedoc """ A connection. @@ -170,11 +144,14 @@ defmodule Cream.Connection do ## Options - * `ttl` (`integer`) - Apply time to live (expiry) to the item. Default `nil`. + * `ttl` (`integer | nil`) - Apply time to live (expiry) to the item. Default + `nil`. * `return_cas` (`boolean`) - Return cas value in result. Default `false`. + * `coder` (`atom | nil`) - Use a `Cream.Coder` on value. Overrides the config + used by `start_link/1`. Default `nil`. - If `ttl` is set explicitly on the item, that will take precedence over - the `ttl` specified in `opts`. + If `ttl` is set explicitly on the item, that will take precedence over the + `ttl` specified in `opts`. ## Examples @@ -354,8 +331,9 @@ defmodule Cream.Connection do %{socket: socket} = state {key, value, ttl, cas} = item + coder = opts[:coder] || state.coder - with {:ok, value, flags} <- encode(value, state), + with {:ok, value, flags} <- encode(value, coder), packet = Protocol.set({key, value, ttl, cas, flags}), :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), @@ -379,6 +357,7 @@ defmodule Cream.Connection do %{socket: socket} = state packet = Protocol.get(key) + coder = opts[:coder] || state.coder with :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), @@ -387,7 +366,7 @@ defmodule Cream.Connection do do case packet.status do :ok -> - with {:ok, value} <- decode(packet.value, packet.extras.flags, state) do + with {:ok, value} <- decode(packet.value, packet.extras.flags, coder) do if opts[:return_cas] do {:reply, {:ok, value, packet.cas}, state} else @@ -422,14 +401,17 @@ defmodule Cream.Connection do {:disconnect, :tcp_closed, state} end - defp encode(value, %{coder: nil}), do: {:ok, value, 0} - defp encode(value, %{coder: coder}) do - {coder.encode(value), 0} + defp encode(value, nil), do: {:ok, value, 0} + defp encode(value, coder) do + Coder.encode(coder, value, 0) end - defp decode(value, _flags, %{coder: nil}), do: {:ok, value} - defp decode(value, flags, %{coder: coder}) do - coder.decode(value, flags) + defp decode(value, _flags, nil), do: {:ok, value} + defp decode(value, flags, coders) when is_list(coders) do + Enum.reverse(coders) |> Coder.decode(value, flags) + end + defp decode(value, flags, coder) do + Coder.decode(coder, value, flags) end end diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex index 8eccf45..8d78de0 100644 --- a/lib/cream/protocol.ex +++ b/lib/cream/protocol.ex @@ -1,4 +1,5 @@ defmodule Cream.Protocol do + @moduledoc false @request 0x80 @response 0x81 diff --git a/lib/cream/utils.ex b/lib/cream/utils.ex index 90011e7..1eb9807 100644 --- a/lib/cream/utils.ex +++ b/lib/cream/utils.ex @@ -1,4 +1,5 @@ defmodule Cream.Utils do + @moduledoc false def normalize_item(item, opts \\ []) diff --git a/test/coder_test.exs b/test/coder_test.exs index 836b213..a4f1fb1 100644 --- a/test/coder_test.exs +++ b/test/coder_test.exs @@ -4,35 +4,35 @@ defmodule Coder.Test do alias Cream.Coder test "jason" do - {:ok, json, 0b1} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0) + {:ok, json, 0b1} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0) {:ok, %{"foo" => "bar"}} = Jason.decode(json) - {:ok, json, 0b101} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0b100) + {:ok, json, 0b101} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0b100) {:ok, %{"foo" => "bar"}} = Jason.decode(json) - {:error, _reason} = Coder.apply_encode(Coder.Jason, {"foo", "bar"}, 0b100) + {:error, _reason} = Coder.encode(Coder.Jason, {"foo", "bar"}, 0b100) - {:ok, json, flags} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0) - {:ok, %{"foo" => "bar"}} = Coder.apply_decode(Coder.Jason, json, flags) + {:ok, json, flags} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0) + {:ok, %{"foo" => "bar"}} = Coder.decode(Coder.Jason, json, flags) - {:ok, json, flags} = Coder.apply_encode(Coder.Jason, %{"foo" => "bar"}, 0b100) - {:ok, %{"foo" => "bar"}} = Coder.apply_decode(Coder.Jason, json, flags) + {:ok, json, flags} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0b100) + {:ok, %{"foo" => "bar"}} = Coder.decode(Coder.Jason, json, flags) end test "gzip" do - {:ok, zipped, 0b10} = Coder.apply_encode(Coder.Gzip, "foobar", 0) + {:ok, zipped, 0b10} = Coder.encode(Coder.Gzip, "foobar", 0) "foobar" = :zlib.gunzip(zipped) - {:ok, zipped, 0b110} = Coder.apply_encode(Coder.Gzip, "foobar", 0b100) + {:ok, zipped, 0b110} = Coder.encode(Coder.Gzip, "foobar", 0b100) "foobar" = :zlib.gunzip(zipped) - {:error, _reason} = Coder.apply_encode(Coder.Gzip, :foobar, 0) + {:error, _reason} = Coder.encode(Coder.Gzip, :foobar, 0) - {:ok, zipped, flags} = Coder.apply_encode(Coder.Gzip, "foobar", 0) - {:ok, "foobar"} = Coder.apply_decode(Coder.Gzip, zipped, flags) + {:ok, zipped, flags} = Coder.encode(Coder.Gzip, "foobar", 0) + {:ok, "foobar"} = Coder.decode(Coder.Gzip, zipped, flags) - {:ok, zipped, flags} = Coder.apply_encode(Coder.Gzip, "foobar", 0b100) - {:ok, "foobar"} = Coder.apply_decode(Coder.Gzip, zipped, flags) + {:ok, zipped, flags} = Coder.encode(Coder.Gzip, "foobar", 0b100) + {:ok, "foobar"} = Coder.decode(Coder.Gzip, zipped, flags) end test "jason + gzip" do @@ -41,17 +41,17 @@ defmodule Coder.Test do term = %{"foo" => "bar"} - {:ok, data, 0b11} = Coder.apply_encode(encoders, term, 0) + {:ok, data, 0b11} = Coder.encode(encoders, term, 0) ^term = data |> :zlib.gunzip() |> Jason.decode!() - {:ok, data, 0b111} = Coder.apply_encode(encoders, term, 0b100) + {:ok, data, 0b111} = Coder.encode(encoders, term, 0b100) ^term = data |> :zlib.gunzip() |> Jason.decode!() - {:ok, data, 0b11} = Coder.apply_encode(encoders, term, 0) - {:ok, ^term} = Coder.apply_decode(decoders, data, 0b11) + {:ok, data, 0b11} = Coder.encode(encoders, term, 0) + {:ok, ^term} = Coder.decode(decoders, data, 0b11) - {:ok, data, flags} = Coder.apply_encode(encoders, term, 0b100) - {:ok, ^term} = Coder.apply_decode(decoders, data, flags) + {:ok, data, flags} = Coder.encode(encoders, term, 0b100) + {:ok, ^term} = Coder.decode(decoders, data, flags) end end diff --git a/test/connection_test.exs b/test/connection_test.exs index 34a6a54..27fae39 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -1,7 +1,7 @@ defmodule ConnectionTest do use ExUnit.Case - alias Cream.Connection + alias Cream.{Connection, Coder} setup_all do {:ok, conn} = Connection.start_link() @@ -31,4 +31,33 @@ defmodule ConnectionTest do assert is_integer(cas) end + # Note that this uses Cream.Coder.Json which is different from + # Cream.Coder.Jason and only exists in test env. + test "single coder", %{conn: conn} do + coder = Coder.Json + map = %{"a" => "b"} + json = ~S({"a":"b"}) + + # Encode maps. + :ok = Connection.set(conn, {"foo", map}, coder: coder) + {:ok, ^map} = Connection.get(conn, "foo", coder: coder) + {:ok, ^json} = Connection.get(conn, "foo") + + # Doesn't encode binaries. + :ok = Connection.set(conn, {"foo", json}, coder: coder) + {:ok, ^json} = Connection.get(conn, "foo", coder: coder) + {:ok, ^json} = Connection.get(conn, "foo") + end + + test "double coder", %{conn: conn} do + coder = [Coder.Jason, Coder.Gzip] + map = %{"a" => "b"} + + :ok = Connection.set(conn, {"foo", map}, coder: coder) + {:ok, ^map} = Connection.get(conn, "foo", coder: coder) + + {:ok, data} = Connection.get(conn, "foo") + {:ok, ^map} = :zlib.gunzip(data) |> Jason.decode() + end + end diff --git a/test/support/coder/json.ex b/test/support/coder/json.ex new file mode 100644 index 0000000..3b4c431 --- /dev/null +++ b/test/support/coder/json.ex @@ -0,0 +1,26 @@ +defmodule Cream.Coder.Json do + use Bitwise + + @behaviour Cream.Coder + + def encode(value, flags) when is_binary(value) do + {:ok, value, flags &&& 0b00} + end + + def encode(value, flags) do + with {:ok, json} <- Jason.encode(value) do + {:ok, json, flags ||| 0b01} + end + end + + def decode(value, flags) when (flags &&& 0b01) == 0b01 do + with {:ok, json} <- Jason.decode(value) do + {:ok, json} + end + end + + def decode(value, _flags) do + {:ok, value} + end + +end From e9350447059be47541564fa24001c9c91cc8be70 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Mon, 5 Jul 2021 16:56:55 -0500 Subject: [PATCH 08/28] Basic cluster support, Ruby compat tests --- .gitignore | 4 ++- README.md | 4 ++- lib/cream/cluster.ex | 60 +++++++++++++++++++++++++++++++++++++-- lib/cream/connection.ex | 5 ++++ lib/cream/continuum.ex | 9 +++--- test/ruby_compat_test.exs | 36 +++++++++++++++++++++++ test/support/populate.rb | 23 +++++++-------- 7 files changed, 120 insertions(+), 21 deletions(-) create mode 100644 test/ruby_compat_test.exs diff --git a/.gitignore b/.gitignore index b21d4b0..d16a285 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ erl_crash.dump .bundle -.elixir_ls \ No newline at end of file +.elixir_ls + +vendor/bundle \ No newline at end of file diff --git a/README.md b/README.md index 838885b..3dea131 100644 --- a/README.md +++ b/README.md @@ -239,8 +239,10 @@ Test dependencies: Then run... ``` -bundle install docker-compose up -d +bundle install --path vendor/bundle +bundle exec ruby test/support/populate.rb + mix test # Stop and clean up containers diff --git a/lib/cream/cluster.ex b/lib/cream/cluster.ex index f46ae72..2198bfe 100644 --- a/lib/cream/cluster.ex +++ b/lib/cream/cluster.ex @@ -1,8 +1,64 @@ defmodule Cream.Cluster do - defstruct [:continuum, :servers] + @moduledoc false - def set(item) do + defstruct [:continuum, :servers, :connections] + alias Cream.{Connection, Continuum} + + def new(config \\ []) do + case Keyword.fetch!(config, :servers) do + [server] -> + config = Keyword.put(config, :server, server) + {:ok, conn} = Connection.start_link(config) + %__MODULE__{servers: [server], connections: {conn}} + + servers -> + continuum = Continuum.new(servers) + + conns = Enum.map(servers, fn server -> + config = Keyword.put(config, :server, server) + {:ok, conn} = Connection.start_link(config) + conn + end) + |> List.to_tuple() + + %__MODULE__{servers: servers, connections: conns, continuum: continuum} + end + end + + def set(cluster, item, opts \\ []) do + key = elem(item, 0) + find_conn(cluster, key) + |> Cream.Connection.set(item, opts) + end + + def get(cluster, key, opts \\ []) do + find_conn(cluster, key) + |> Cream.Connection.get(key, opts) + end + + def flush(cluster, opts \\ []) do + errors = cluster.servers + |> Enum.with_index() + |> Enum.reduce(%{}, fn {server, i}, acc -> + conn = elem(cluster.connections, i) + case Cream.Connection.flush(conn, opts) do + :ok -> acc + {:error, reason} -> Map.put(acc, server, reason) + end + end) + + if errors == %{} do + :ok + else + {:errors, errors} + end + end + + def find_conn(%{continuum: nil, connections: {conn}}, _key), do: conn + def find_conn(%{continuum: continuum, connections: conns}, key) do + {:ok, server_id} = Continuum.find(continuum, key) + elem(conns, server_id) end end diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 420d804..9359b24 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -2,6 +2,9 @@ defmodule Cream.Connection do @moduledoc """ Basic connection to a single memcached server. + You probably don't want this unless you're making some low level Memcached + client. See `Cream.Client` instead. + ## Global configuration You can globally configure _all connections_ via `Config`. @@ -14,6 +17,8 @@ defmodule Cream.Connection do Now every single connection will use `"foo.bar.com:11211"` for `:server` unless overwritten by an argument passed to `start_link/1` or `child_spec/1`. + > **IMPORTANT!** This will affect connections made by `Cream.Client`. + ## Reconnecting `Cream.Connection` uses the awesome `Connection` behaviour and also "active" diff --git a/lib/cream/continuum.ex b/lib/cream/continuum.ex index d808714..c0dde28 100644 --- a/lib/cream/continuum.ex +++ b/lib/cream/continuum.ex @@ -7,12 +7,13 @@ defmodule Cream.Continuum do total_servers = length(servers) total_weight = length(servers) # TODO implement weights - Enum.reduce(servers, [], fn server, acc -> + Enum.with_index(servers) + |> Enum.reduce([], fn {server, server_index}, acc -> count = entry_count_for(server, 1, total_servers, total_weight) Enum.reduce 0..count-1, acc, fn i, acc -> - hash = :crypto.hash(:sha, "#{server}:#{i}") |> Base.encode16 + hash = :crypto.hash(:sha, "#{server}:#{i}") |> Base.encode16() {value, _} = hash |> String.slice(0, 8) |> Integer.parse(16) - [{server, value} | acc] + [{server_index, value} | acc] end end) |> Enum.sort_by(fn {_id, value} -> value end) @@ -53,7 +54,7 @@ defmodule Cream.Continuum do defp binary_search(entries, value), do: binary_search(entries, value, 0, tuple_size(entries)-1) defp binary_search(_entries, _value, lower, upper) when lower > upper, do: upper defp binary_search(entries, value, lower, upper) do - i = ((lower + upper) / 2) |> trunc + i = ((lower + upper) / 2) |> trunc() { _, candidate_value } = elem(entries, i) cond do candidate_value == value -> i diff --git a/test/ruby_compat_test.exs b/test/ruby_compat_test.exs new file mode 100644 index 0000000..24152eb --- /dev/null +++ b/test/ruby_compat_test.exs @@ -0,0 +1,36 @@ +defmodule RubyCompatTest do + use ExUnit.Case + + alias Cream.{Cluster, Coder} + + setup_all do + cluster = Cluster.new(servers: [ + "localhost:11201", + "localhost:11202", + "localhost:11203" + ]) + + [cluster: cluster] + end + + test "no coder", %{cluster: cluster} do + Enum.each(1..99, fn i -> + expected = to_string(i) + {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_ruby_test_key_#{i}") + + expected = Jason.encode!(%{value: i}) + {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_json_test_key_#{i}") + end) + end + + test "jason coder", %{cluster: cluster} do + Enum.each(1..99, fn i -> + expected = i + {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_ruby_test_key_#{i}", coder: Coder.Jason) + + expected = %{"value" => i} + {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_json_test_key_#{i}", coder: Coder.Jason) + end) + end + +end diff --git a/test/support/populate.rb b/test/support/populate.rb index d8f46b8..a3f40ce 100644 --- a/test/support/populate.rb +++ b/test/support/populate.rb @@ -1,18 +1,15 @@ require "dalli" require "json" -client = Dalli::Client.new( - [ - "localhost:11201", - "localhost:11202", - "localhost:11203" - ], - serializer: JSON, -) +servers = [ + "localhost:11201", + "localhost:11202", + "localhost:11203" +] -if ARGV[0] == "json" - client.set "foo", {"one" => ["two", "three"]} - exit(0) -end +client = Dalli::Client.new(servers, serializer: JSON) -20.times{ |i| client.set("cream_ruby_test_key_#{i}", i) } +100.times do |i| + client.set("cream_ruby_test_key_#{i}", i) + client.set("cream_json_test_key_#{i}", {value: i}) +end \ No newline at end of file From 3626f86af6a53509f25c6682132b081535a2bc67 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Fri, 23 Jul 2021 19:41:31 -0500 Subject: [PATCH 09/28] Client, tests, docs --- README.md | 232 ++++------------------------------ config/test.exs | 9 +- lib/cream/client.ex | 240 ++++++++++++++++++++++++++++++++---- lib/cream/coder.ex | 4 +- lib/cream/connection.ex | 14 ++- mix.exs | 7 +- test/client_test.exs | 36 ++++++ test/coder_test.exs | 2 +- test/ruby_compat_test.exs | 4 +- test/support/test_client.ex | 7 ++ 10 files changed, 305 insertions(+), 250 deletions(-) create mode 100644 test/client_test.exs create mode 100644 test/support/test_client.ex diff --git a/README.md b/README.md index 3dea131..a180d69 100644 --- a/README.md +++ b/README.md @@ -2,231 +2,43 @@ A Dalli compatible memcached client. -It uses the same consistent hashing algorithm to connect to a cluster of memcached servers. - -## Table of contents - -1. [Features](#features) -1. [Installation](#installation) -1. [Quickstart](#quickstart) -1. [Connecting to a cluster](#connecting-to-a-cluster) -1. [Using modules](#using-modules) -1. [Memcachex options](#memcachex-options) -1. [Memcachex API](#memcachex-api) -1. [Ruby compatibility](#ruby-compatibility) -1. [Supervision](#supervision) -1. [Instrumentation](#instrumentation) -1. [Documentation](https://hexdocs.pm/cream/Cream.Cluster.html) -1. [Running the tests](#running-the-tests) -1. [TODO](#todo) - -## Features - - * connect to a "cluster" of memcached servers - * compatible with Ruby's Dallie gem (same consistent hashing algorithm) - * fetch with anonymous function - * multi set - * multi get - * multi fetch - * built in pooling via [poolboy](https://github.com/devinus/poolboy) - * complete supervision trees - * [fully documented](https://hexdocs.pm/cream/Cream.Cluster.html) - * instrumentation with the [Instrumentation](https://hexdocs.pm/instrumentation) package. - - -## Installation - -In your `mix.exs` file... - -```elixir -def deps do - [ - {:cream, ">= 0.1.0"} - ] -end -``` +It uses the same consistent hashing algorithm as Dalli to determine which server +a key is on. ## Quickstart -```elixir -# Connects to localhost:11211 with worker pool of size 10 -{:ok, cluster} = Cream.Cluster.start_link - -# Single set and get -Cream.Cluster.set(cluster, {"name", "Callie"}) -Cream.Cluster.get(cluster, "name") -# => "Callie" - -# Single fetch -Cream.Cluster.fetch cluster, "some", fn -> - "thing" -end -# => "thing" - -# Multi set / multi get with list -Cream.Cluster.set(cluster, [{"name", "Callie"}, {"buddy", "Chris"}]) -Cream.Cluster.get(cluster, ["name", "buddy"]) -# => %{"name" => "Callie", "buddy" => "Chris"} - -# Multi set / multi get with map -Cream.Cluster.set(cluster, %{"species" => "canine", "gender" => "female"}) -Cream.Cluster.get(cluster, ["species", "gender"]) -# => %{"species" => "canine", "gender" => "female"} +Sensible defaults... -# Multi fetch -Cream.Cluster.fetch cluster, ["foo", "bar", "baz"], fn missing_keys -> - Enum.map(missing_keys, &String.reverse/1) -end -# => %{"foo" => "oof", "bar" => "rab", "baz" => "zab"} -``` - -## Connecting to a cluster - -```elixir -{:ok, cluster} = Cream.Cluster.start_link servers: ["cache01:11211", "cache02:11211"] ``` +iex(1)> {:ok, client} = Cream.Client.start_link() +{:ok, #PID<0.265.0>} -## Using modules - -You can use modules to configure clusters, exactly like how Ecto repos work. - -```elixir -# In config/*.exs - -config :my_app, MyCluster, - servers: ["cache01:11211", "cache02:11211"], - pool: 5 +iex(1)> Cream.Client.set(client, {"foo", "bar"}) +:ok -# Elsewhere - -defmodule MyCluster do - use Cream.Cluster, otp_app: :my_app - - # Optional callback to do runtime configuration. - def init(config) do - # config = Keyword.put(config, :pool, System.get_env("POOL_SIZE")) - {:ok, config} - end -end - -MyCluster.start_link -MyCluster.get("foo") +iex(1)> Cream.Client.get(client, "foo") +{:ok, "bar"} ``` -## Memcachex options +As a module with custom config... -Cream uses Memcachex for individual connections to the cluster. You can pass -options to Memcachex via `Cream.Cluster.start_link/1`: - -```elixir -Cream.Cluster.start_link( - servers: ["localhost:11211"], - memcachex: [ttl: 3600, namespace: "foo"] -) ``` +import Config -Or if using modules: - -```elixir -use Mix.Config - -config :my_app, MyCluster, - servers: ["localhost:11211"], - memcachex: [ttl: 3600, namespace: "foo"] +config MyClient, servers: ["memcached01:11211", "memcached02:11211"] -MyCluster.start_link -``` - -Any option you can pass to -[`Memcache.start_link`](https://hexdocs.pm/memcachex/Memcache.html#start_link/2), -you can pass via the `:memcachex` option for `Cream.Cluster.start_link`. - -## Memcachex API - -`Cream.Cluster`'s API is very small: `get`, `set`, `fetch`, `flush`. It may -expand in the future, but for now, you can access Memcachex's API directly -if you need. - -Cream will still provide worker pooling and key routing, even when using -Memcachex's API directly. - -If you are using a single key, things are pretty straight forward... - -```elixir -results = Cream.Cluster.with_conn cluster, key, fn conn -> - Memcache.get(conn, key) +def MyClient do + use Cream.Client end -``` - -It gets a bit more complex with a list of keys... - -```elixir -results = Cream.Cluster.with_conn cluster, keys, fn conn, keys -> - Memcache.multi_get(conn, keys) -end -# results will be a list of whatever was returned by the invocations of the given function. -``` - -Basically, Cream will group keys by memcached server and then call the provided -function for each group and return a list of the results of each call. - -## Ruby compatibility -By default, Dalli uses Marshal to encode values stored in memcached, which -Elixir can't understand. So you have to change the serializer to something like JSON: +iex(1)> {:ok, _client} = MyClient.start_link() +{:ok, #PID<0.265.0>} -Ruby -```ruby -client = Dalli::Client.new( - ["host01:11211", "host2:11211"], - serializer: JSON, -) -client.set("foo", 100) -``` - -Elixir -```elixir -{:ok, cluster} = Cream.Cluster.start_link( - servers: ["host01:11211", "host2:11211"], - memcachex: [coder: Memcache.Coder.JSON] -) -Cream.Cluster.get(cluster, "foo") -# => "100" -``` - -So now both Ruby and Elixir will read/write to the memcached cluster in JSON, -but still beware! There are some differences between how Ruby and Elixir parse -JSON. For example, if you write an integer with Ruby, Ruby will read an integer, -but Elixir will read a string. - -## Supervision - -Everything is supervised, even the supervisors, so it really does make a -supervision tree. - -A "cluster" is really a poolboy pool of cluster supervisors. A cluster -supervisor supervises each `Memcache.Connection` process and one -`Cream.Cluster.Worker` process. +iex(1)> MyClient.set({"foo", "bar"}) +:ok -No pids are stored anywhere, but instead processes are tracked via Elixir's -`Registry` module. - -The results of `Cream.Cluster.start_link` and `MyClusterModule.start_link` can -be inserted into your application's supervision tree. - -## Instrumentation - -Cream uses [Instrumentation](https://hexdocs.pm/instrumentation) for... well, -instrumentation. It's default logging is hooked into this package. You can do -your own logging (or instrumentation) very easily. - -```elixir -config :my_app, MyCluster, - log: false - -Instrumentation.subscribe "cream", fn tag, payload -> - Logger.debug("cream.#{tag} took #{payload[:duration]} ms") -end +iex(1)> MyClient.get(client, "foo") +{:ok, "bar"} ``` ## Running the tests @@ -238,7 +50,7 @@ Test dependencies: * Bundler Then run... -``` +```sh docker-compose up -d bundle install --path vendor/bundle bundle exec ruby test/support/populate.rb @@ -250,7 +62,7 @@ docker-compose stop docker-compose rm ``` -## TODO +## Todo * Server weights * Parallel memcached requests diff --git a/config/test.exs b/config/test.exs index 3a3ce8b..a5b8d32 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,8 +1,5 @@ -use Mix.Config +import Config -config :cream, Test.Cluster, - servers: ["localhost:11201", "localhost:11202", "localhost:11203"], - memcachex: [coder: Memcache.Coder.JSON] +config :cream, TestClient, coder: Cream.Coder.Jason -config :logger, - level: :info +config :logger, :level, :info diff --git a/lib/cream/client.ex b/lib/cream/client.ex index c637718..c9a2107 100644 --- a/lib/cream/client.ex +++ b/lib/cream/client.ex @@ -1,64 +1,256 @@ defmodule Cream.Client do + @moduledoc """ + Pooled connections to a Memcached server or cluster. + + This is usually what you want instead of `Cream.Connection`, even if you have + just a single Memcached server. + + `Cream.Client` uses `NimblePool` for connection pooling (lazy by default). + + ## Global configuration + + You can globally configure _all clients_ via `Config`. + + ``` + import Config + + config :cream, Cream.Client, servers: [ + "localhost:11211", + "localhost:11212", + "localhost:11213" + ] + ``` + + Now every single client will those servers unless overwritten by an argument + passed to `start_link/1` or `child_spec/1`. + + ## Using as a module + + You can `use Cream.Config` for a higher level of convenience. + + ``` + def MyClient do + use Cream.Client, coder: Cream.Coder.Jason + end + ``` + + Start it directly... + + ``` + {:ok, _client} = MyClient.start_link() + ``` + + Or as a part of a supervision tree... + + ``` + children = [MyClient] + Supervisor.start_link(children, strategy: :one_for_one) + ``` + + Config is merged in a sensible order... + 1. `use` args + 1. `c:config/0` + 1. `c:start_link/1` args + + See `c:config/0` for an example. + """ + + @typedoc """ + A `Cream.Client`. + """ + @type t :: GenServer.server() + + @typedoc """ + Error reason. + """ + @type reason :: binary | atom | term + + alias Cream.{Cluster} + @behaviour NimblePool @defaults [ pool_size: 5, lazy: true, - servers: ["localhost:11211"] + servers: ["localhost:11211"], + coder: nil, + ttl: nil ] + + @doc """ + Default config. + + ``` + #{inspect @defaults, pretty: true, width: 0} + ``` + + * `pool_size` - How big the connection pool is. + * `lazy` - If the connection pool is lazily loaded. + * `servers` - What memcached servers to connect to. + * `coder` - What `Cream.Coder` to use. + * `ttl` - Default time to live (expiry) in seconds to use with `set/3`. + """ def defaults, do: @defaults + @doc """ + `Config` merged with `defaults/0`. + + ``` + import Config + config :cream, Cream.Client, coder: FooCoder, ttl: 60 + + iex(1)> Cream.Client.config() + #{inspect Keyword.merge(@defaults, coder: FooCoder, ttl: 60), pretty: true, width: 0} + ``` + """ def config do - Keyword.merge( - @defaults, - Application.get_application(__MODULE__) - |> Application.get_env(__MODULE__, []) - ) + config = Application.get_application(__MODULE__) + |> Application.get_env(__MODULE__, []) + + Keyword.merge(@defaults, config) end @impl NimblePool def init_pool(config) do - config = Map.new(config) - |> Map.merge(%{ - continuum: Cream.Continuum.new(config[:servers]) - }) - {:ok, config} end @impl NimblePool def init_worker(config) do - worker = Map.new(config[:servers], fn server -> - {:ok, conn} = Cream.Connection.start_link(server: server) - {server, conn} - end) - - {:ok, worker, config} + {:ok, Cluster.new(config), config} end @impl NimblePool - def handle_checkout(:checkout, _from, worker, config) do - client = Map.put(config, :connections, worker) - {:ok, client, worker, config} + def handle_checkout(:checkout, _from, cluster, config) do + {:ok, cluster, cluster, config} end + @doc false + def checkout(pool, f) do + NimblePool.checkout!(pool, :checkout, fn _, client -> + {f.(client), client} + end) + end + + @doc """ + Child specification for supervisors. + + `config` will be merged over `config/0`. + """ def child_spec(config \\ []) do config = Keyword.merge(config(), config) Keyword.take(config, [:pool_size, :lazy]) |> Keyword.put(:worker, {__MODULE__, config}) + |> Keyword.put(:name, config[:name]) |> NimblePool.child_spec() end + @doc """ + Start a client. + + `config` will be merged over `config/0`. + + See `defaults/0` for valid config options. + """ def start_link(config \\ []) do %{start: {m, f, a}} = child_spec(config) apply(m, f, a) end - def checkout(pool, f) do - NimblePool.checkout!(pool, :checkout, fn _, client -> - {f.(client), client} - end) + def get(client, key, opts \\ []) do + checkout(client, &Cluster.get(&1, key, opts)) + end + + def set(client, item, opts \\ []) do + checkout(client, &Cluster.set(&1, item, opts)) + end + + def flush(client, opts \\ []) do + checkout(client, &Cluster.flush(&1, opts)) + end + + @doc """ + `Config` merged with `config/0` and `use` args. + + ``` + import Config + + config :cream, Cream.Client, servers: ["memcached:11211"] + config :my_app, MyClient, coder: Cream.Coder.Jason + + defmodule MyClient do + use Cream.Client, ttl: 60 + end + + iex(1)> Cream.Client.config() + #{inspect Keyword.merge(@defaults, + servers: ["memcached:11211"]), + pretty: true, + width: 0 + } + + iex(1)> MyClient.config() + #{inspect Keyword.merge(@defaults, + servers: ["memcached:11211"], + coder: Cream.Coder.Jason, + ttl: 60), + pretty: true, + width: 0 + } + ``` + """ + @callback config :: Keyword.t + + @doc """ + Start a client. + + `config` is merged with `c:config/0`. + + See `defaults/0` for valid `config` options. + """ + @callback start_link(config :: Keyword.t) :: {:ok, t} | {:error, reason} + + @doc """ + Child specification for supervisors. + + `config` is merged with `c:config/0`. + + See `defaults/0` for valid `config` options. + """ + @callback child_spec(config :: Keyword.t) :: Supervisor.child_spec + + defmacro __using__(config \\ []) do + quote do + @config unquote(config) + + def config do + Cream.Client.config() + |> Keyword.merge(@config) + |> Keyword.merge(config_config()) + end + + def child_spec(config \\ []) do + Keyword.merge(config(), config) + |> Keyword.put(:name, __MODULE__) + |> Cream.Client.child_spec() + end + + def start_link(config \\ []) do + %{start: {m, f, a}} = child_spec(config) + apply(m, f, a) + end + + def get(key, opts \\ []) do + Cream.Client.get(__MODULE__, key, opts) + end + + defp config_config do + Application.get_application(__MODULE__) + |> Application.get_env(__MODULE__, []) + end + + end end end diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index 001adfc..502c17b 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -41,8 +41,8 @@ defmodule Cream.Coder do end ``` - Notice we set a bit (`0b1`) on flags to indicate the value is serialized. When - the value is not serialized, we unset the bit. + Notice we set a the bit `0b1` on flags to indicate the value is serialized. + When the value is not serialized, we unset the bit. Now let's see the coder in action. We use two connections, one that uses the coder and one that doesn't. diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 9359b24..d0aed8b 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -260,7 +260,7 @@ defmodule Cream.Connection do state = %{ config: config, socket: nil, - coder: nil, + coder: config[:coder], errors: 0, } @@ -336,7 +336,7 @@ defmodule Cream.Connection do %{socket: socket} = state {key, value, ttl, cas} = item - coder = opts[:coder] || state.coder + coder = resolve_coder(opts, state) with {:ok, value, flags} <- encode(value, coder), packet = Protocol.set({key, value, ttl, cas, flags}), @@ -362,7 +362,7 @@ defmodule Cream.Connection do %{socket: socket} = state packet = Protocol.get(key) - coder = opts[:coder] || state.coder + coder = resolve_coder(opts, state) with :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), @@ -406,6 +406,14 @@ defmodule Cream.Connection do {:disconnect, :tcp_closed, state} end + defp resolve_coder(opts, state) do + if Keyword.has_key?(opts, :coder) do + opts[:coder] + else + state.coder + end + end + defp encode(value, nil), do: {:ok, value, 0} defp encode(value, coder) do Coder.encode(coder, value, 0) diff --git a/mix.exs b/mix.exs index 89ce9bb..0702411 100644 --- a/mix.exs +++ b/mix.exs @@ -3,8 +3,8 @@ defmodule Cream.Mixfile do def project do [app: :cream, - version: "0.2.0", - elixir: "~> 1.4", + version: "1.0.0", + elixir: "~> 1.0", build_embedded: Mix.env == :prod, start_permanent: Mix.env == :prod, elixirc_paths: elixirc_paths(Mix.env), @@ -23,6 +23,9 @@ defmodule Cream.Mixfile do extras: [ "README.md": [title: "README"], "CHANGELOG.md": [title: "CHANGELOG"] + ], + groups_for_modules: [ + "Coders": [Cream.Coder.Gzip, Cream.Coder.Jason] ] ] ] diff --git a/test/client_test.exs b/test/client_test.exs new file mode 100644 index 0000000..c2770e0 --- /dev/null +++ b/test/client_test.exs @@ -0,0 +1,36 @@ +defmodule ClientTest do + use ExUnit.Case + + setup_all do + {:ok, _client} = TestClient.start_link() + :ok + end + + test "Config gets merged with use args" do + config = TestClient.config() + + assert config[:servers] == ~w(localhost:11201 localhost:11202 localhost:11203) + assert config[:coder] == Cream.Coder.Jason + end + + test ":coder arg overrides config" do + Enum.each(0..99, fn i -> + expected = to_string(i) + {:ok, ^expected} = TestClient.get("cream_ruby_test_key_#{i}", coder: nil) + + expected = Jason.encode!(%{value: i}) + {:ok, ^expected} = TestClient.get("cream_json_test_key_#{i}", coder: nil) + end) + end + + test "json coder" do + Enum.each(0..99, fn i -> + expected = i + {:ok, ^expected} = TestClient.get("cream_ruby_test_key_#{i}") + + expected = %{"value" => i} + {:ok, ^expected} = TestClient.get("cream_json_test_key_#{i}") + end) + end + +end diff --git a/test/coder_test.exs b/test/coder_test.exs index a4f1fb1..3429958 100644 --- a/test/coder_test.exs +++ b/test/coder_test.exs @@ -1,4 +1,4 @@ -defmodule Coder.Test do +defmodule CoderTest do use ExUnit.Case alias Cream.Coder diff --git a/test/ruby_compat_test.exs b/test/ruby_compat_test.exs index 24152eb..01953a1 100644 --- a/test/ruby_compat_test.exs +++ b/test/ruby_compat_test.exs @@ -14,7 +14,7 @@ defmodule RubyCompatTest do end test "no coder", %{cluster: cluster} do - Enum.each(1..99, fn i -> + Enum.each(0..99, fn i -> expected = to_string(i) {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_ruby_test_key_#{i}") @@ -24,7 +24,7 @@ defmodule RubyCompatTest do end test "jason coder", %{cluster: cluster} do - Enum.each(1..99, fn i -> + Enum.each(0..99, fn i -> expected = i {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_ruby_test_key_#{i}", coder: Coder.Jason) diff --git a/test/support/test_client.ex b/test/support/test_client.ex new file mode 100644 index 0000000..5171ab4 --- /dev/null +++ b/test/support/test_client.ex @@ -0,0 +1,7 @@ +defmodule TestClient do + use Cream.Client, servers: [ + "localhost:11201", + "localhost:11202", + "localhost:11203" + ] +end From e0791352e655044d54c2678228f1e8143676f2fd Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Fri, 23 Jul 2021 20:49:52 -0500 Subject: [PATCH 10/28] Delete command, more API for client/cluster --- lib/cream/client.ex | 24 +++++++++++++++++ lib/cream/cluster.ex | 10 +++++++ lib/cream/connection.ex | 56 ++++++++++++++++++++++++++++++++++++++++ lib/cream/protocol.ex | 34 ++++++++++++++++++------ test/client_test.exs | 12 +++++++++ test/connection_test.exs | 27 +++++++++++++++++++ 6 files changed, 155 insertions(+), 8 deletions(-) diff --git a/lib/cream/client.ex b/lib/cream/client.ex index c9a2107..92a3c93 100644 --- a/lib/cream/client.ex +++ b/lib/cream/client.ex @@ -166,6 +166,14 @@ defmodule Cream.Client do checkout(client, &Cluster.set(&1, item, opts)) end + def fetch(client, key, opts \\ [], f) do + checkout(client, &Cluster.fetch(&1, key, opts, f)) + end + + def delete(client, key, opts \\ []) do + checkout(client, &Cluster.delete(&1, key, opts)) + end + def flush(client, opts \\ []) do checkout(client, &Cluster.flush(&1, opts)) end @@ -245,6 +253,22 @@ defmodule Cream.Client do Cream.Client.get(__MODULE__, key, opts) end + def set(item, opts \\ []) do + Cream.Client.set(__MODULE__, item, opts) + end + + def fetch(key, opts \\ [], f) do + Cream.Client.fetch(__MODULE__, key, opts, f) + end + + def delete(key, opts \\ []) do + Cream.Client.delete(__MODULE__, key, opts) + end + + def flush(opts \\ []) do + Cream.Client.flush(__MODULE__, opts) + end + defp config_config do Application.get_application(__MODULE__) |> Application.get_env(__MODULE__, []) diff --git a/lib/cream/cluster.ex b/lib/cream/cluster.ex index 2198bfe..81c6d4a 100644 --- a/lib/cream/cluster.ex +++ b/lib/cream/cluster.ex @@ -37,6 +37,16 @@ defmodule Cream.Cluster do |> Cream.Connection.get(key, opts) end + def fetch(cluster, key, opts \\ [], f) do + find_conn(cluster, key) + |> Cream.Connection.fetch(key, opts, f) + end + + def delete(cluster, key, opts \\ []) do + find_conn(cluster, key) + |> Cream.Connection.delete(key, opts) + end + def flush(cluster, opts \\ []) do errors = cluster.servers |> Enum.with_index() diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index d0aed8b..e1d67f9 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -144,6 +144,11 @@ defmodule Cream.Connection do Connection.call(conn, {:flush, opts}) end + @spec delete(t, binary, Keyword.t) :: :ok | {:error, reason} + def delete(conn, key, opts \\ []) do + Connection.call(conn, {:delete, key, opts}) + end + @doc """ Set a single item. @@ -233,6 +238,35 @@ defmodule Cream.Connection do end end + @doc """ + Get a key and set its value if it doesn't exist. + + If `key` doesn't exist, then `f` is used to generate its value and set it on the server. + + :ok = flush(conn) + {:ok, nil} = get("foo") + {:ok, "bar"} = fetch(conn, "foo", fn -> "bar" end) + {:ok, "bar"} = get("foo") + + `set` options will be used if the key doesn't exist. `get` options should work regardless. + """ + @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:ok, term, cas} | {:error, reason} + def fetch(conn, key, options \\ [], f) do + case get(conn, key, Keyword.put(options, :verbose, true)) do + {:ok, value} -> {:ok, value} + {:ok, value, cas} -> {:ok, value, cas} + {:error, :not_found} -> + value = f.() + case set(conn, {key, value}, options) do + :ok -> {:ok, value} + {:ok, cas} -> {:ok, value, cas} + error -> error + end + + error -> error + end + end + @doc """ Send a noop command to server. @@ -332,6 +366,28 @@ defmodule Cream.Connection do end end + def handle_call({:delete, key, options}, _from, state) do + %{socket: socket} = state + + with :ok <- :inet.setopts(socket, active: false), + :ok <- :gen_tcp.send(socket, Protocol.delete(key)), + {:ok, packet} <- Protocol.recv_packet(socket), + :ok <- :inet.setopts(socket, active: true) + do + case packet.status do + :ok -> {:reply, :ok, state} + :not_found -> if options[:verbose] do + {:reply, {:error, :not_found}, state} + else + {:reply, :ok, state} + end + error -> {:reply, {:error, error}, state} + end + else + {:error, reason} -> {:disconnect, reason, state} + end + end + def handle_call({:set, item, opts}, _from, state) do %{socket: socket} = state diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex index 8d78de0..95841e3 100644 --- a/lib/cream/protocol.ex +++ b/lib/cream/protocol.ex @@ -4,14 +4,15 @@ defmodule Cream.Protocol do @request 0x80 @response 0x81 - @get 0x00 - @set 0x01 - @flush 0x08 - @getq 0x09 - @noop 0x0a - @getk 0x0c - @getkq 0x0d - @setq 0x11 + @get 0x00 + @set 0x01 + @delete 0x04 + @flush 0x08 + @getq 0x09 + @noop 0x0a + @getk 0x0c + @getkq 0x0d + @setq 0x11 [ {0x0000, :ok}, @@ -71,6 +72,23 @@ defmodule Cream.Protocol do ] end + def delete(key) do + key_size = byte_size(key) + + [ + <<@request::bytes(1)>>, + <<@delete::bytes(1)>>, + <>, + <<0x00::bytes(1)>>, + <<0x00::bytes(1)>>, + <<0x00::bytes(2)>>, + <>, + <<0x00::bytes(4)>>, + <<0x00::bytes(8)>>, + key + ] + end + def set({key, value, ttl, cas, flags}) do key_size = byte_size(key) value_size = byte_size(value) diff --git a/test/client_test.exs b/test/client_test.exs index c2770e0..b453de7 100644 --- a/test/client_test.exs +++ b/test/client_test.exs @@ -33,4 +33,16 @@ defmodule ClientTest do end) end + test "fetch" do + :ok = TestClient.delete("foo") + + {:error, :not_found} = TestClient.get("foo", verbose: true) + {:ok, "bar"} = TestClient.fetch("foo", fn -> "bar" end) + {:ok, "bar"} = TestClient.get("foo") + + :ok = TestClient.set({"foo", "baz"}) + {:ok, "baz"} = TestClient.fetch("foo", fn -> "bar" end) + {:ok, "baz"} = TestClient.get("foo") + end + end diff --git a/test/connection_test.exs b/test/connection_test.exs index 27fae39..90ce7c9 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -31,6 +31,33 @@ defmodule ConnectionTest do assert is_integer(cas) end + test "delete", %{conn: conn} do + {:error, :not_found} = Connection.delete(conn, "foo", verbose: true) + :ok = Connection.delete(conn, "foo") + + :ok = Connection.set(conn, {"foo", "bar"}) + {:ok, "bar"} = Connection.get(conn, "foo") + :ok = Connection.delete(conn, "foo", verbose: true) + {:error, :not_found} = Connection.get(conn, "foo", verbose: true) + {:error, :not_found} = Connection.delete(conn, "foo", verbose: true) + + :ok = Connection.set(conn, {"foo", "bar"}) + {:ok, "bar"} = Connection.get(conn, "foo") + :ok = Connection.delete(conn, "foo") + {:error, :not_found} = Connection.get(conn, "foo", verbose: true) + {:error, :not_found} = Connection.delete(conn, "foo", verbose: true) + end + + test "fetch", %{conn: conn} do + {:error, :not_found} = Connection.get(conn, "foo", verbose: true) + {:ok, "bar"} = Connection.fetch(conn, "foo", fn -> "bar" end) + {:ok, "bar"} = Connection.get(conn, "foo") + + :ok = Connection.set(conn, {"foo", "baz"}) + {:ok, "baz"} = Connection.fetch(conn, "foo", fn -> "bar" end) + {:ok, "baz"} = Connection.get(conn, "foo") + end + # Note that this uses Cream.Coder.Json which is different from # Cream.Coder.Jason and only exists in test env. test "single coder", %{conn: conn} do From 176bdf0058e5af715a2a8e1464d72b6a6e5200e4 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Fri, 23 Jul 2021 20:52:00 -0500 Subject: [PATCH 11/28] Typo --- lib/cream/cluster.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cream/cluster.ex b/lib/cream/cluster.ex index 81c6d4a..f015d5b 100644 --- a/lib/cream/cluster.ex +++ b/lib/cream/cluster.ex @@ -61,7 +61,7 @@ defmodule Cream.Cluster do if errors == %{} do :ok else - {:errors, errors} + {:error, errors} end end From 85cfea153e5b82356f7f043c3ac4e81f608d2640 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Fri, 23 Jul 2021 20:57:31 -0500 Subject: [PATCH 12/28] Words --- lib/cream/client.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cream/client.ex b/lib/cream/client.ex index 92a3c93..ea07422 100644 --- a/lib/cream/client.ex +++ b/lib/cream/client.ex @@ -21,8 +21,8 @@ defmodule Cream.Client do ] ``` - Now every single client will those servers unless overwritten by an argument - passed to `start_link/1` or `child_spec/1`. + Now every single client will use those servers unless overwritten by an + argument passed to `start_link/1` or `child_spec/1`. ## Using as a module From 37048d89ebd7afbf872f676b60c1fc2cf4099005 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Fri, 23 Jul 2021 21:18:19 -0500 Subject: [PATCH 13/28] Doc stuff --- lib/cream/client.ex | 20 ++++++++++++++------ mix.exs | 3 +-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/cream/client.ex b/lib/cream/client.ex index ea07422..9d285c9 100644 --- a/lib/cream/client.ex +++ b/lib/cream/client.ex @@ -2,10 +2,13 @@ defmodule Cream.Client do @moduledoc """ Pooled connections to a Memcached server or cluster. - This is usually what you want instead of `Cream.Connection`, even if you have - just a single Memcached server. + `NimblePool` is used for connection pooling (lazy by default). - `Cream.Client` uses `NimblePool` for connection pooling (lazy by default). + ## Quickstart + + {:ok, client} = Cream.Client.start_link() + :ok = Cream.Client.set(client, {"foo", "bar"}) + {:ok, "bar"} = Cream.Client.get(client, "foo") ## Global configuration @@ -21,12 +24,12 @@ defmodule Cream.Client do ] ``` - Now every single client will use those servers unless overwritten by an - argument passed to `start_link/1` or `child_spec/1`. + Now _every_ client will use those servers unless overwritten by an argument + passed to `start_link/1` or `child_spec/1`. ## Using as a module - You can `use Cream.Config` for a higher level of convenience. + You can `use Cream.Client` for a higher level of convenience. ``` def MyClient do @@ -34,6 +37,11 @@ defmodule Cream.Client do end ``` + Configure with `Config`... + ``` + config :my_app, MyClient, servers: ["cache.myapp.com"] + ``` + Start it directly... ``` diff --git a/mix.exs b/mix.exs index 0702411..d0eda5f 100644 --- a/mix.exs +++ b/mix.exs @@ -19,9 +19,8 @@ defmodule Cream.Mixfile do ], docs: [ - main: "readme", + main: "Cream.Client", extras: [ - "README.md": [title: "README"], "CHANGELOG.md": [title: "CHANGELOG"] ], groups_for_modules: [ From ec1cceefc038a34e933fe00951245bed3e77a5eb Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Fri, 23 Jul 2021 21:36:58 -0500 Subject: [PATCH 14/28] Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66732a3..239381e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # cream changes +## 1.0.0 +----------- +* Simplified API; no more multi-get or multi-set (subject to change if need arises). +* Lazy connection pooling via `NimblePool`. +* Increase compatibility with Dalli via serialization that is aware of flags. +* Use `telemetry` for instrumentation. +* Drop dependency on `memcachex`. + ## 0.2.0 ----------- * Integrate with `Instrumentation` package From 88b7cc39a05c16dc4f23f8b31e9c06cd1961dfe3 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Thu, 11 Aug 2022 19:14:36 -0500 Subject: [PATCH 15/28] Use errors, fix connection_test --- config/config.exs | 30 +-- config/dev.exs | 2 +- lib/cream/coder.ex | 3 +- lib/cream/connection.ex | 352 +++++++++++++++------------ lib/cream/errors/connection_error.ex | 25 ++ lib/cream/errors/error.ex | 18 ++ lib/cream/errors/multi_error.ex | 3 + lib/cream/item.ex | 11 + lib/cream/protocol.ex | 4 +- mix.lock | 10 +- test/connection_test.exs | 49 +++- 11 files changed, 298 insertions(+), 209 deletions(-) create mode 100644 lib/cream/errors/connection_error.ex create mode 100644 lib/cream/errors/error.ex create mode 100644 lib/cream/errors/multi_error.ex create mode 100644 lib/cream/item.ex diff --git a/config/config.exs b/config/config.exs index ba97a87..9def7c2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,31 +1,3 @@ -# This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. -use Mix.Config - -# This configuration is loaded before any dependency and is restricted -# to this project. If another project depends on this project, this -# file won't be loaded nor affect the parent project. For this reason, -# if you want to provide default values for your application for -# 3rd-party users, it should be done in your "mix.exs" file. - -# You can configure for your application as: -# -# config :cream, key: :value -# -# And access this configuration in your application as: -# -# Application.get_env(:cream, :key) -# -# Or configure a 3rd-party app: -# -# config :logger, level: :info -# - -# It is also possible to import configuration files, relative to this -# directory. For example, you can emulate configuration per environment -# by uncommenting the line below and defining dev.exs, test.exs and such. -# Configuration from the imported file will override the ones defined -# here (which is why it is important to import them last). -# +import Config import_config "#{Mix.env}.exs" diff --git a/config/dev.exs b/config/dev.exs index 92f459e..6e0dee9 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :cream, servers: ["localhost:11211", "127.0.0.1:11211"] diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index 502c17b..1f38346 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -121,7 +121,8 @@ defmodule Cream.Coder do @doc false def decode(coders, value, flags) when is_list(coders) do - Enum.reduce_while(coders, {:ok, value}, fn coder, {:ok, value} -> + Enum.reverse(coders) + |> Enum.reduce_while({:ok, value}, fn coder, {:ok, value} -> case decode(coder, value, flags) do {:ok, value} -> {:cont, {:ok, value}} {:error, reason} -> {:halt, {:error, reason}} diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index e1d67f9..68ee04b 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -40,7 +40,7 @@ defmodule Cream.Connection do use Connection require Logger - alias Cream.{Protocol, Coder} + alias Cream.{Protocol, Coder, Error, ConnectionError} @typedoc """ A connection. @@ -112,13 +112,26 @@ defmodule Cream.Connection do Note that this will typically _always_ return `{:ok conn}`, even if the actual TCP connection failed. If that is the case, subsequent uses of the connection - will return `{:error, :not_connected}`. + will return `{:error, %Cream.ConnectionError{}}`. - ``` - {:ok, conn} = #{inspect __MODULE__}.start_link() + # Connect to localhost:11211 + + iex> start_link() + {:ok, conn} + + # Connect to a specific server + + iex> start_link(server: "memcache.foo.com:11211") + {:ok, conn} + + # Connection error + + iex> start_link(server: "localhost:99899") + {:ok, conn} + + iex> get(conn, "foo") + {:error, %Cream.Error{reason: :econnrefused, server: "localhost:99899"}} - {:ok, conn} = #{inspect __MODULE__}.start_link(server: "memcache.foo.com:11211") - ``` """ @spec start_link(Keyword.t) :: {:ok, t} | {:error, reason} def start_link(config \\ []) do @@ -129,24 +142,48 @@ defmodule Cream.Connection do @doc """ Clear all keys. - ``` - # Flush immediately - iex(1)> #{inspect __MODULE__}.flush(conn) - :ok + # Flush immediately + iex> #{inspect __MODULE__}.flush(conn) + :ok + + # Flush after 60 seconds + iex> #{inspect __MODULE__}.flush(conn, ttl: 60) + :ok - # Flush after 60 seconds - iex(1)> #{inspect __MODULE__}.flush(conn, ttl: 60) - :ok - ``` """ - @spec flush(t, Keyword.t) :: :ok | {:error, reason} + @spec flush(t, Keyword.t) :: :ok | {:error, Error.t} | {:error, ConnectionError.t} def flush(conn, opts \\ []) do - Connection.call(conn, {:flush, opts}) + case Connection.call(conn, {:flush, opts}) do + {:ok, packet} -> case packet.status do + :ok -> :ok + reason -> {:error, Error.exception(reason)} + end + %ConnectionError{} = conn_error -> {:error, conn_error} + end end - @spec delete(t, binary, Keyword.t) :: :ok | {:error, reason} + @doc """ + Delete a key. + + iex> delete(conn, "foo") + :ok + + iex> delete(conn, "foo", verbose: true) + {:error, %Cream.Error{reason: :not_found}} + + """ + @spec delete(t, binary, Keyword.t) :: :ok | {:error, Error.t} | {:error, ConnectionError.t} def delete(conn, key, opts \\ []) do - Connection.call(conn, {:delete, key, opts}) + case Connection.call(conn, {:delete, key}) do + {:ok, %{status: :ok}} -> :ok + {:ok, %{status: :not_found}} -> if opts[:verbose] do + {:error, Error.exception(:not_found)} + else + :ok + end + {:error, %ConnectionError{} = conn_error} -> {:error, conn_error} + {:error, reason} -> {:error, Error.exception(reason)} + end end @doc """ @@ -154,51 +191,56 @@ defmodule Cream.Connection do ## Options - * `ttl` (`integer | nil`) - Apply time to live (expiry) to the item. Default - `nil`. - * `return_cas` (`boolean`) - Return cas value in result. Default `false`. - * `coder` (`atom | nil`) - Use a `Cream.Coder` on value. Overrides the config + * `:cas` - `(boolean)` - Return cas value in result. Default `false`. + * `:coder` - `(atom|nil)` - Use a `Cream.Coder` on value. Overrides the config used by `start_link/1`. Default `nil`. - If `ttl` is set explicitly on the item, that will take precedence over the - `ttl` specified in `opts`. + `:ttl` on the item is in seconds. See `t:item/0` for more info. ## Examples - ``` - # Basic set - iex(1)> set(conn, {"foo", "bar"}) - :ok + # Basic set + iex> set(conn, {"foo", "bar"}) + :ok - # Use ttl on item. - iex(1)> set(conn, {"foo", "bar", ttl: 60}) - :ok + # Use ttl. + iex> set(conn, {"foo", "bar", ttl: 60}) + :ok - # Use ttl from opts. - iex(1)> set(conn, {"foo", "bar"}, ttl: 60) - :ok + # Set using cas. + iex> set(conn, {"foo", "bar", cas: 122}) + :ok - # Return cas value. - iex(1)> set(conn, {"foo", "bar"}, return_cas: true) - {:ok, 123} + # Return cas value. + iex> set(conn, {"foo", "bar"}, cas: true) + {:ok, 123} - # Set using bad cas value results in error. - iex(1)> set(conn, {"foo", "bar", cas: 124}) - {:error, :exists} + # Set using bad cas value results in error. + iex> set(conn, {"foo", "bar", cas: 124}) + {:error, %Cream.Error{reason: :exists}} - # Set using cas value and return new cas value. - iex(1)> set(conn, {"foo", "bar", cas: 123}, return_cas: true) - {:ok, 124} + # Set using cas value and return new cas value. + iex> set(conn, {"foo", "bar", cas: 123}, cas: true) + {:ok, 124} - # Set using cas. - iex(1)> set(conn, {"foo", "bar", cas: 124}) - :ok - ``` """ - @spec set(t, item, Keyword.t) :: :ok | {:ok, cas} | {:error, reason} + @spec set(t, item, Keyword.t) :: :ok | {:ok, cas} | {:error, Error.t} | {:error, ConnectionError.t} def set(conn, item, opts \\ []) do - item = Cream.Utils.normalize_item(item) - Connection.call(conn, {:set, item, opts}) + with {:ok, item} <- Cream.Item.from_args(item), + {:ok, conn_coder} <- Connection.call(conn, :get_coder), + {:ok, item} <- encode(item, opts[:coder], conn_coder), + {:ok, %{status: :ok} = packet} <- Connection.call(conn, {:set, item}) + do + if opts[:cas] do + {:ok, packet.cas} + else + :ok + end + else + {:ok, %{status: status}} -> {:error, Error.exception(status)} + {:error, %ConnectionError{} = conn_error} -> {:error, conn_error} + {:error, reason} -> {:error, Error.exception(reason)} + end end @doc """ @@ -206,35 +248,44 @@ defmodule Cream.Connection do ## Options - * `verbose` (boolean, default false) Missing value will return `{:error, :not_found}`. - * `cas` (boolean, default false) Return cas value in result. + * `:verbose` - `(boolean)` - If `true`, missing keys will return an error. If + `false`, missing keys will return `nil`. Default `false`. + * `:cas` - `(boolean)` - Return cas value in result. Default `false`. ## Examples - ``` - iex(1)> get(conn, "foo") - {:ok, nil} + iex> get(conn, "foo") + {:ok, nil} - iex(1)> get(conn, "foo", verbose: true) - {:error, :not_found} + iex> get(conn, "foo", verbose: true) + {:error, %Cream.Error{reason: :not_found}} - iex(1)> get(conn, "name") - {:ok, "Callie"} + iex> get(conn, "name") + {:ok, "Callie"} + + iex> get(conn, "name", cas: true) + {:ok, "Callie", 123} - iex(1)> get(conn, "name", return_cas: true) - {:ok, "Callie", 123} - ``` """ - @spec get(t, binary, Keyword.t) :: {:ok, term} | {:ok, term, cas} | {:error, reason} - def get(conn, key, options \\ []) do - case Connection.call(conn, {:get, key, options}) do - {:error, :not_found} = result -> if options[:verbose] do - result + @spec get(t, binary, Keyword.t) :: {:ok, term} | {:ok, term, cas} | {:error, Error.t} | {:error, ConnectionError.t} + def get(conn, key, opts \\ []) do + with {:ok, %{status: :ok} = packet, coder} <- Connection.call(conn, {:get, key}), + {:ok, value} <- decode(packet, opts[:coder], coder) + do + if opts[:cas] do + {:ok, value, packet.cas} + else + {:ok, value} + end + else + {:ok, %{status: :not_found}, _coder} -> if opts[:verbose] do + {:error, Error.exception(:not_found)} else {:ok, nil} end - - result -> result + {:ok, %{status: reason}, _coder} -> {:error, Error.exception(reason)} + {:error, %ConnectionError{} = conn_error} -> {:error, conn_error} + {:error, reason} -> {:error, Error.exception(reason)} end end @@ -243,21 +294,23 @@ defmodule Cream.Connection do If `key` doesn't exist, then `f` is used to generate its value and set it on the server. - :ok = flush(conn) - {:ok, nil} = get("foo") - {:ok, "bar"} = fetch(conn, "foo", fn -> "bar" end) - {:ok, "bar"} = get("foo") + iex> :ok = flush(conn) + iex> {:ok, nil} = get(conn, "foo") + iex> {:ok, "bar"} = fetch(conn, "foo", fn -> "bar" end) + iex> {:ok, "bar"} = get(conn, "foo") - `set` options will be used if the key doesn't exist. `get` options should work regardless. + `set/3` options will be used if the key doesn't exist. + + `get/3` options will always be used. """ - @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:ok, term, cas} | {:error, reason} - def fetch(conn, key, options \\ [], f) do - case get(conn, key, Keyword.put(options, :verbose, true)) do + @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:ok, term, cas} | {:error, Error.t} | {:error, ConnectionError.t} + def fetch(conn, key, opts \\ [], f) do + case get(conn, key, Keyword.put(opts, :verbose, true)) do {:ok, value} -> {:ok, value} {:ok, value, cas} -> {:ok, value, cas} - {:error, :not_found} -> + {:error, %Error{reason: :not_found}} -> value = f.() - case set(conn, {key, value}, options) do + case set(conn, {key, value}, opts) do :ok -> {:ok, value} {:ok, cas} -> {:ok, value, cas} error -> error @@ -271,22 +324,23 @@ defmodule Cream.Connection do Send a noop command to server. You can use this to see if you are connected to the server. - ``` - iex(1)> noop(conn) - :ok - iex(1)> noop(conn) - {:error, :not_connected} - ``` + iex> noop(conn) + :ok + + iex> noop(conn) + {:error, %Cream.Error{reason: :econnrefused, server: "localhost:11211"}} + """ - @spec noop(t) :: :ok | {:error, reason} + @spec noop(t) :: :ok | {:error, Error.t} | {:error, ConnectionError.t} def noop(conn) do case Connection.call(conn, :noop) do {:ok, packet} -> case packet do %{status: :ok} -> :ok - %{status: reason} -> {:error, reason} + %{status: reason} -> {:error, Error.exception(reason)} end - error -> error + {:error, %ConnectionError{} = conn_error} -> {:error, conn_error} + {:error, reason} -> {:error, Error.exception(reason)} end end @@ -295,7 +349,8 @@ defmodule Cream.Connection do config: config, socket: nil, coder: config[:coder], - errors: 0, + err_reason: nil, + err_count: 0, } {:connect, :init, state} @@ -318,16 +373,16 @@ defmodule Cream.Connection do %{usec: System.monotonic_time(:microsecond) - start}, %{context: context, server: server} ) - {:ok, %{state | socket: socket, errors: 0}} + {:ok, %{state | socket: socket, err_count: 0}} {:error, reason} -> - errors = state.errors + 1 + count = state.err_count + 1 :telemetry.execute( [:cream, :connection, :error], %{usec: System.monotonic_time(:microsecond) - start}, - %{context: context, reason: reason, server: server, count: errors} + %{context: context, reason: reason, server: server, count: count} ) - {:backoff, 1000, %{state | errors: errors}} + {:backoff, 1000, %{state | err_reason: reason, err_count: count}} end end @@ -345,11 +400,16 @@ defmodule Cream.Connection do {:connect, reason, %{state | socket: nil}} end + def handle_call(:get_coder, _from, state) do + {:reply, {:ok, state.coder}, state} + end + def handle_call(_command, _from, state) when is_nil(state.socket) do - {:reply, {:error, :not_connected}, state} + error = ConnectionError.exception(state.err_reason, state.config[:server]) + {:reply, {:error, error}, state} end - def handle_call({:flush, options}, _from, state) do + def handle_call({:flush, options}, from, state) do %{socket: socket} = state with :ok <- :inet.setopts(socket, active: false), @@ -357,16 +417,13 @@ defmodule Cream.Connection do {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do - case packet.status do - :ok -> {:reply, :ok, state} - error -> {:reply, {:error, error}, state} - end + {:reply, {:ok, packet}, state} else - {:error, reason} -> {:disconnect, reason, state} + {:error, reason} -> handle_conn_error(reason, from, state) end end - def handle_call({:delete, key, options}, _from, state) do + def handle_call({:delete, key}, from, state) do %{socket: socket} = state with :ok <- :inet.setopts(socket, active: false), @@ -374,73 +431,41 @@ defmodule Cream.Connection do {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do - case packet.status do - :ok -> {:reply, :ok, state} - :not_found -> if options[:verbose] do - {:reply, {:error, :not_found}, state} - else - {:reply, :ok, state} - end - error -> {:reply, {:error, error}, state} - end + {:reply, {:ok, packet}, state} else - {:error, reason} -> {:disconnect, reason, state} + {:error, reason} -> handle_conn_error(reason, from, state) end end - def handle_call({:set, item, opts}, _from, state) do + def handle_call({:set, item}, from, state) do %{socket: socket} = state - {key, value, ttl, cas} = item - coder = resolve_coder(opts, state) + packet = Protocol.set(item) - with {:ok, value, flags} <- encode(value, coder), - packet = Protocol.set({key, value, ttl, cas, flags}), - :ok <- :inet.setopts(socket, active: false), + with :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do - case packet.status do - :ok -> if opts[:return_cas] do - {:reply, {:ok, packet.cas}, state} - else - {:reply, :ok, state} - end - error -> {:reply, {:error, error}, state} - end + {:reply, {:ok, packet}, state} else - {:error, reason} -> {:disconnect, reason, state} + {:error, reason} -> handle_conn_error(reason, from, state) end end - def handle_call({:get, key, opts}, _from, state) do - %{socket: socket} = state + def handle_call({:get, key}, from, state) do + %{socket: socket, coder: coder} = state packet = Protocol.get(key) - coder = resolve_coder(opts, state) with :ok <- :inet.setopts(socket, active: false), :ok <- :gen_tcp.send(socket, packet), {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do - case packet.status do - :ok -> - with {:ok, value} <- decode(packet.value, packet.extras.flags, coder) do - if opts[:return_cas] do - {:reply, {:ok, value, packet.cas}, state} - else - {:reply, {:ok, value}, state} - end - else - {:error, reason} -> {:reply, {:error, reason}, state} - end - - reason -> {:reply, {:error, reason}, state} - end + {:reply, {:ok, packet, coder}, state} else - {:error, reason} -> {:disconnect, reason, state} + {:error, reason} -> handle_conn_error(reason, from, state) end end @@ -462,25 +487,34 @@ defmodule Cream.Connection do {:disconnect, :tcp_closed, state} end - defp resolve_coder(opts, state) do - if Keyword.has_key?(opts, :coder) do - opts[:coder] - else - state.coder - end + defp handle_conn_error(reason, from, %{config: config} = state) do + error = ConnectionError.exception(reason: reason, server: config[:server]) + :ok = GenServer.reply(from, {:error, error}) + {:disconnect, reason, state} end - defp encode(value, nil), do: {:ok, value, 0} - defp encode(value, coder) do - Coder.encode(coder, value, 0) - end + defp encode(%Cream.Item{} = item, opts_coder, conn_coder) do + result = case {opts_coder, conn_coder} do + {false, _} -> :noop + {nil, nil} -> :noop + {coder, _} when coder != nil -> Coder.encode(coder, item.value, item.flags) + {_, coder} when coder != nil -> Coder.encode(coder, item.value, item.flags) + end - defp decode(value, _flags, nil), do: {:ok, value} - defp decode(value, flags, coders) when is_list(coders) do - Enum.reverse(coders) |> Coder.decode(value, flags) + case result do + :noop -> {:ok, item} + {:ok, value, flags} -> {:ok, %{item | value: value, flags: flags}} + error -> error + end end - defp decode(value, flags, coder) do - Coder.decode(coder, value, flags) + + defp decode(packet, opts_coder, conn_coder) do + case {opts_coder, conn_coder} do + {false, _} -> {:ok, packet.value} + {nil, nil} -> {:ok, packet.value} + {coder, _} when coder != nil -> Coder.decode(coder, packet.value, packet.extras.flags) + {_, coder} when coder != nil -> Coder.decode(coder, packet.value, packet.extras.flags) + end end end diff --git a/lib/cream/errors/connection_error.ex b/lib/cream/errors/connection_error.ex new file mode 100644 index 0000000..14f7e81 --- /dev/null +++ b/lib/cream/errors/connection_error.ex @@ -0,0 +1,25 @@ +defmodule Cream.ConnectionError do + defexception [:reason, :server] + + @type t :: %__MODULE__{reason: atom, server: binary} + + @impl Exception + + def exception(fields) when is_list(fields) do + struct!(__MODULE__, fields) + end + + def exception(reason, server) do + %__MODULE__{reason: reason, server: server} + end + + @impl Exception + + def message(e) do + case e do + %{reason: reason, server: nil} -> to_string(reason) + %{reason: reason, server: server} -> "#{reason} - #{server}" + end + end + +end diff --git a/lib/cream/errors/error.ex b/lib/cream/errors/error.ex new file mode 100644 index 0000000..c8d15cf --- /dev/null +++ b/lib/cream/errors/error.ex @@ -0,0 +1,18 @@ +defmodule Cream.Error do + defexception [:reason] + + @type t :: %__MODULE__{reason: atom} + + @impl Exception + + def exception(reason) when is_atom(reason) do + %__MODULE__{reason: reason} + end + + @impl Exception + + def message(e) do + to_string(e.reason) + end + +end diff --git a/lib/cream/errors/multi_error.ex b/lib/cream/errors/multi_error.ex new file mode 100644 index 0000000..ad43533 --- /dev/null +++ b/lib/cream/errors/multi_error.ex @@ -0,0 +1,3 @@ +defmodule Cream.MultiError do + defexception [:keys] +end diff --git a/lib/cream/item.ex b/lib/cream/item.ex new file mode 100644 index 0000000..76dd57d --- /dev/null +++ b/lib/cream/item.ex @@ -0,0 +1,11 @@ +defmodule Cream.Item do + defstruct [:key, :value, ttl: 0, cas: 0, flags: 0] + + def from_args({key, value}) do + {:ok, %__MODULE__{key: key, value: value}} + end + + def from_args({key, value, opts}) do + {:ok, struct!(__MODULE__, Keyword.merge(opts, [key: key, value: value]))} + end +end diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex index 95841e3..c15d701 100644 --- a/lib/cream/protocol.ex +++ b/lib/cream/protocol.ex @@ -89,7 +89,9 @@ defmodule Cream.Protocol do ] end - def set({key, value, ttl, cas, flags}) do + def set(item) do + %{key: key, value: value, ttl: ttl, cas: cas, flags: flags} = item + key_size = byte_size(key) value_size = byte_size(value) body_size = key_size + value_size + 8 diff --git a/mix.lock b/mix.lock index 5be89b7..4461b20 100644 --- a/mix.lock +++ b/mix.lock @@ -1,12 +1,12 @@ %{ "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, - "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, + "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "nimble_pool": {:hex, :nimble_pool, "0.2.4", "1db8e9f8a53d967d595e0b32a17030cdb6c0dc4a451b8ac787bf601d3f7704c3", [:mix], [], "hexpm", "367e8071e137b787764e6a9992ccb57b276dc2282535f767a07d881951ebeac6"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, } diff --git a/test/connection_test.exs b/test/connection_test.exs index 90ce7c9..bef9a06 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -1,7 +1,8 @@ defmodule ConnectionTest do use ExUnit.Case + # doctest Cream.Connection, import: true, only: [set: 3] - alias Cream.{Connection, Coder} + alias Cream.{Connection, Error, Coder} setup_all do {:ok, conn} = Connection.start_link() @@ -14,42 +15,42 @@ defmodule ConnectionTest do test "set", %{conn: conn} do :ok = Connection.set(conn, {"foo", "bar"}) - {:ok, cas} = Connection.set(conn, {"foo", "bar"}, return_cas: true) - {:error, :exists} = Connection.set(conn, {"foo", "bar1", cas: cas+1}) + {:ok, cas} = Connection.set(conn, {"foo", "bar"}, cas: true) + {:error, %Error{reason: :exists}} = Connection.set(conn, {"foo", "bar1", cas: cas+1}) {:ok, "bar"} = Connection.get(conn, "foo") :ok = Connection.set(conn, {"foo", "bar1", cas: cas}) {:ok, "bar1"} = Connection.get(conn, "foo") end test "get", %{conn: conn} do - :ok = Connection.set(conn, {"name", "Callie"}) + {:ok, set_cas} = Connection.set(conn, {"name", "Callie"}, cas: true) {:ok, nil} = Connection.get(conn, "foo") - {:error, :not_found} = Connection.get(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) {:ok, "Callie"} = Connection.get(conn, "name") - {:ok, "Callie", cas} = Connection.get(conn, "name", return_cas: true) - assert is_integer(cas) + {:ok, "Callie", get_cas} = Connection.get(conn, "name", cas: true) + assert set_cas == get_cas end test "delete", %{conn: conn} do - {:error, :not_found} = Connection.delete(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", verbose: true) :ok = Connection.delete(conn, "foo") :ok = Connection.set(conn, {"foo", "bar"}) {:ok, "bar"} = Connection.get(conn, "foo") :ok = Connection.delete(conn, "foo", verbose: true) - {:error, :not_found} = Connection.get(conn, "foo", verbose: true) - {:error, :not_found} = Connection.delete(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", verbose: true) :ok = Connection.set(conn, {"foo", "bar"}) {:ok, "bar"} = Connection.get(conn, "foo") :ok = Connection.delete(conn, "foo") - {:error, :not_found} = Connection.get(conn, "foo", verbose: true) - {:error, :not_found} = Connection.delete(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", verbose: true) end test "fetch", %{conn: conn} do - {:error, :not_found} = Connection.get(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) {:ok, "bar"} = Connection.fetch(conn, "foo", fn -> "bar" end) {:ok, "bar"} = Connection.get(conn, "foo") @@ -87,4 +88,26 @@ defmodule ConnectionTest do {:ok, ^map} = :zlib.gunzip(data) |> Jason.decode() end + test "override coder" do + value = %{"a" => "b"} + {:ok, conn} = Connection.start_link(coder: Coder.Jason) + + :ok = Connection.set(conn, {"foo", value}) + :ok = Connection.set(conn, {"bar", value}, coder: [Coder.Jason, Coder.Gzip]) + + {:ok, foo} = Connection.get(conn, "foo", coder: false) + {:ok, bar} = Connection.get(conn, "bar", coder: false) + + assert is_binary(foo) + assert is_binary(bar) + + assert foo != value + assert bar != value + + assert Jason.decode!(foo) == value + + assert :zlib.gunzip(bar) != value + assert :zlib.gunzip(bar) |> Jason.decode!() == value + end + end From 77975e03991839b880d27aff8ebe20e74da64869 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 13 Aug 2022 14:14:53 -0500 Subject: [PATCH 16/28] All tests passing --- .tool-versions | 1 + Gemfile.lock | 4 ++-- config/runtime.exs | 32 +++++++++++++++++++++++++++ docker-compose.yml | 15 ++++++++----- lib/cream/client.ex | 10 ++++++--- lib/cream/coder.ex | 3 +-- lib/cream/connection.ex | 44 ++++++++++++++++++++++++------------- mix.exs | 2 ++ mix.lock | 6 +++++ test/client_test.exs | 11 ++++++---- test/ruby_compat_test.exs | 36 ------------------------------ test/support/populate.rb | 10 ++++----- test/support/test_client.ex | 6 +---- 13 files changed, 102 insertions(+), 78 deletions(-) create mode 100644 .tool-versions create mode 100644 config/runtime.exs delete mode 100644 test/ruby_compat_test.exs diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..974865f --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +ruby 2.7.6 diff --git a/Gemfile.lock b/Gemfile.lock index 09383fe..c45e055 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: https://rubygems.org/ specs: - dalli (2.7.6) + dalli (3.2.2) PLATFORMS ruby @@ -10,4 +10,4 @@ DEPENDENCIES dalli BUNDLED WITH - 1.15.1 + 2.1.4 diff --git a/config/runtime.exs b/config/runtime.exs new file mode 100644 index 0000000..135eb28 --- /dev/null +++ b/config/runtime.exs @@ -0,0 +1,32 @@ +import Config + +{:ok, _apps} = Application.ensure_all_started(:finch) +{:ok, _pid} = Finch.start_link(name: Finch) + +{:ok, resp} = Finch.build(:get, "http://localhost/containers/json", [], nil, unix_socket: "/var/run/docker.sock") +|> Finch.request(Finch) + +servers = Jason.decode!(resp.body) +|> Enum.reduce([], fn container, acc -> + name = Enum.find_value(container["Names"], fn name -> + if String.starts_with?(name, "/cream-ex-memcached-") do + name + else + false + end + end) + + if name do + [%{"PublicPort" => port}] = container["Ports"] + [{name, port} | acc] + else + acc + end +end) +|> Enum.sort() +|> Enum.map(fn {_name, port} -> "localhost:#{port}" end) + +config :cream, Cream.Connection, server: Enum.at(servers, 0) +config :cream, Cream.Client, servers: servers + +:ok = NimblePool.stop(Finch) diff --git a/docker-compose.yml b/docker-compose.yml index 1b8e832..579ccf6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,17 +2,20 @@ version: "3" services: - memcached_01: + memcached-1: + container_name: cream-ex-memcached-1 image: memcached:alpine ports: - - 11201:11211 + - 11211 - memcached_02: + memcached-2: + container_name: cream-ex-memcached-2 image: memcached:alpine ports: - - 11202:11211 + - 11211 - memcached_03: + memcached-3: + container_name: cream-ex-memcached-3 image: memcached:alpine ports: - - 11203:11211 + - 11211 diff --git a/lib/cream/client.ex b/lib/cream/client.ex index 9d285c9..7432078 100644 --- a/lib/cream/client.ex +++ b/lib/cream/client.ex @@ -241,9 +241,13 @@ defmodule Cream.Client do @config unquote(config) def config do - Cream.Client.config() - |> Keyword.merge(@config) - |> Keyword.merge(config_config()) + base_config = Application.get_env(:cream, Cream.Client, []) + this_config = Application.get_application(__MODULE__) + |> Application.get_env(__MODULE__, []) + + @config + |> Keyword.merge(base_config) + |> Keyword.merge(this_config) end def child_spec(config \\ []) do diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index 1f38346..502c17b 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -121,8 +121,7 @@ defmodule Cream.Coder do @doc false def decode(coders, value, flags) when is_list(coders) do - Enum.reverse(coders) - |> Enum.reduce_while({:ok, value}, fn coder, {:ok, value} -> + Enum.reduce_while(coders, {:ok, value}, fn coder, {:ok, value} -> case decode(coder, value, flags) do {:ok, value} -> {:cont, {:ok, value}} {:error, reason} -> {:halt, {:error, reason}} diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 68ee04b..4b37f18 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -226,9 +226,14 @@ defmodule Cream.Connection do """ @spec set(t, item, Keyword.t) :: :ok | {:ok, cas} | {:error, Error.t} | {:error, ConnectionError.t} def set(conn, item, opts \\ []) do + opts_coder = case opts[:coder] do + false -> false + coder -> List.wrap(coder) + end + with {:ok, item} <- Cream.Item.from_args(item), - {:ok, conn_coder} <- Connection.call(conn, :get_coder), - {:ok, item} <- encode(item, opts[:coder], conn_coder), + {:ok, conn_coder} <- Connection.call(conn, :get_encoder), + {:ok, item} <- encode(item, opts_coder, conn_coder), {:ok, %{status: :ok} = packet} <- Connection.call(conn, {:set, item}) do if opts[:cas] do @@ -269,8 +274,13 @@ defmodule Cream.Connection do """ @spec get(t, binary, Keyword.t) :: {:ok, term} | {:ok, term, cas} | {:error, Error.t} | {:error, ConnectionError.t} def get(conn, key, opts \\ []) do - with {:ok, %{status: :ok} = packet, coder} <- Connection.call(conn, {:get, key}), - {:ok, value} <- decode(packet, opts[:coder], coder) + opts_coder = case opts[:coder] do + false -> false + coder -> List.wrap(coder) |> Enum.reverse() + end + + with {:ok, %{status: :ok} = packet, conn_coder} <- Connection.call(conn, {:get, key}), + {:ok, value} <- decode(packet, opts_coder, conn_coder) do if opts[:cas] do {:ok, value, packet.cas} @@ -345,10 +355,14 @@ defmodule Cream.Connection do end def init(config) do + encoder = List.wrap(config[:coder]) + decoder = Enum.reverse(encoder) + state = %{ config: config, socket: nil, - coder: config[:coder], + encoder: encoder, + decoder: decoder, err_reason: nil, err_count: 0, } @@ -400,8 +414,8 @@ defmodule Cream.Connection do {:connect, reason, %{state | socket: nil}} end - def handle_call(:get_coder, _from, state) do - {:reply, {:ok, state.coder}, state} + def handle_call(:get_encoder, _from, state) do + {:reply, {:ok, state.encoder}, state} end def handle_call(_command, _from, state) when is_nil(state.socket) do @@ -454,7 +468,7 @@ defmodule Cream.Connection do end def handle_call({:get, key}, from, state) do - %{socket: socket, coder: coder} = state + %{socket: socket, decoder: decoder} = state packet = Protocol.get(key) @@ -463,7 +477,7 @@ defmodule Cream.Connection do {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do - {:reply, {:ok, packet, coder}, state} + {:reply, {:ok, packet, decoder}, state} else {:error, reason} -> handle_conn_error(reason, from, state) end @@ -496,9 +510,9 @@ defmodule Cream.Connection do defp encode(%Cream.Item{} = item, opts_coder, conn_coder) do result = case {opts_coder, conn_coder} do {false, _} -> :noop - {nil, nil} -> :noop - {coder, _} when coder != nil -> Coder.encode(coder, item.value, item.flags) - {_, coder} when coder != nil -> Coder.encode(coder, item.value, item.flags) + {[], []} -> :noop + {coder, _} when coder != [] -> Coder.encode(coder, item.value, item.flags) + {_, coder} when coder != [] -> Coder.encode(coder, item.value, item.flags) end case result do @@ -511,9 +525,9 @@ defmodule Cream.Connection do defp decode(packet, opts_coder, conn_coder) do case {opts_coder, conn_coder} do {false, _} -> {:ok, packet.value} - {nil, nil} -> {:ok, packet.value} - {coder, _} when coder != nil -> Coder.decode(coder, packet.value, packet.extras.flags) - {_, coder} when coder != nil -> Coder.decode(coder, packet.value, packet.extras.flags) + {[], []} -> {:ok, packet.value} + {coder, _} when coder != [] -> Coder.decode(coder, packet.value, packet.extras.flags) + {_, coder} when coder != [] -> Coder.decode(coder, packet.value, packet.extras.flags) end end diff --git a/mix.exs b/mix.exs index d0eda5f..843e2d1 100644 --- a/mix.exs +++ b/mix.exs @@ -64,6 +64,8 @@ defmodule Cream.Mixfile do {:connection, "~> 1.0"}, {:nimble_pool, "~> 0.0"}, {:jason, "~> 1.0", only: [:dev, :test]}, + # {:kestrel, "~> 0.0", git: "https://github.com/cjbottaro/kestrel_ex", only: [:dev, :test]}, + {:kestrel, "~> 0.0", path: "~/Projects/kestrel_ex", only: [:dev, :test]}, {:ex_doc, "~> 0.0", only: :dev}, ] end diff --git a/mix.lock b/mix.lock index 4461b20..9abb9de 100644 --- a/mix.lock +++ b/mix.lock @@ -1,11 +1,17 @@ %{ + "castore": {:hex, :castore, "0.1.18", "deb5b9ab02400561b6f5708f3e7660fc35ca2d51bfc6a940d2f513f89c2975fc", [:mix], [], "hexpm", "61bbaf6452b782ef80b33cdb45701afbcf0a918a45ebe7e73f1130d661e66a06"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, + "finch": {:hex, :finch, "0.10.2", "9ad27d68270d879f73f26604bb2e573d40f29bf0e907064a9a337f90a16a0312", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dd8b11b282072cec2ef30852283949c248bd5d2820c88d8acc89402b81db7550"}, + "hpax": {:hex, :hpax, "0.1.1", "2396c313683ada39e98c20a75a82911592b47e5c24391363343bde74f82396ca", [:mix], [], "hexpm", "0ae7d5a0b04a8a60caf7a39fcf3ec476f35cc2cc16c05abea730d3ce6ac6c826"}, "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, + "kestrel": {:git, "https://github.com/cjbottaro/kestrel_ex", "9b651b2d5127c3ee30bac6f80280b08685c15aab", []}, "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, + "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, + "nimble_options": {:hex, :nimble_options, "0.4.0", "c89babbab52221a24b8d1ff9e7d838be70f0d871be823165c94dd3418eea728f", [:mix], [], "hexpm", "e6701c1af326a11eea9634a3b1c62b475339ace9456c1a23ec3bc9a847bca02d"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "nimble_pool": {:hex, :nimble_pool, "0.2.4", "1db8e9f8a53d967d595e0b32a17030cdb6c0dc4a451b8ac787bf601d3f7704c3", [:mix], [], "hexpm", "367e8071e137b787764e6a9992ccb57b276dc2282535f767a07d881951ebeac6"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, diff --git a/test/client_test.exs b/test/client_test.exs index b453de7..6de1ec6 100644 --- a/test/client_test.exs +++ b/test/client_test.exs @@ -1,25 +1,28 @@ defmodule ClientTest do use ExUnit.Case + alias Cream.Error setup_all do {:ok, _client} = TestClient.start_link() + :ok = TestClient.flush() + {_, 0} = System.cmd("bundle", ["exec", "ruby", "test/support/populate.rb"]) :ok end test "Config gets merged with use args" do config = TestClient.config() - assert config[:servers] == ~w(localhost:11201 localhost:11202 localhost:11203) + assert length(config[:servers]) == 3 assert config[:coder] == Cream.Coder.Jason end test ":coder arg overrides config" do Enum.each(0..99, fn i -> expected = to_string(i) - {:ok, ^expected} = TestClient.get("cream_ruby_test_key_#{i}", coder: nil) + {:ok, ^expected} = TestClient.get("cream_ruby_test_key_#{i}", coder: false) expected = Jason.encode!(%{value: i}) - {:ok, ^expected} = TestClient.get("cream_json_test_key_#{i}", coder: nil) + {:ok, ^expected} = TestClient.get("cream_json_test_key_#{i}", coder: false) end) end @@ -36,7 +39,7 @@ defmodule ClientTest do test "fetch" do :ok = TestClient.delete("foo") - {:error, :not_found} = TestClient.get("foo", verbose: true) + {:error, %Error{reason: :not_found}} = TestClient.get("foo", verbose: true) {:ok, "bar"} = TestClient.fetch("foo", fn -> "bar" end) {:ok, "bar"} = TestClient.get("foo") diff --git a/test/ruby_compat_test.exs b/test/ruby_compat_test.exs deleted file mode 100644 index 01953a1..0000000 --- a/test/ruby_compat_test.exs +++ /dev/null @@ -1,36 +0,0 @@ -defmodule RubyCompatTest do - use ExUnit.Case - - alias Cream.{Cluster, Coder} - - setup_all do - cluster = Cluster.new(servers: [ - "localhost:11201", - "localhost:11202", - "localhost:11203" - ]) - - [cluster: cluster] - end - - test "no coder", %{cluster: cluster} do - Enum.each(0..99, fn i -> - expected = to_string(i) - {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_ruby_test_key_#{i}") - - expected = Jason.encode!(%{value: i}) - {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_json_test_key_#{i}") - end) - end - - test "jason coder", %{cluster: cluster} do - Enum.each(0..99, fn i -> - expected = i - {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_ruby_test_key_#{i}", coder: Coder.Jason) - - expected = %{"value" => i} - {:ok, ^expected} = Cream.Cluster.get(cluster, "cream_json_test_key_#{i}", coder: Coder.Jason) - end) - end - -end diff --git a/test/support/populate.rb b/test/support/populate.rb index a3f40ce..108a9cc 100644 --- a/test/support/populate.rb +++ b/test/support/populate.rb @@ -1,11 +1,11 @@ require "dalli" require "json" -servers = [ - "localhost:11201", - "localhost:11202", - "localhost:11203" -] +containers = JSON.parse(`docker compose ps --format json`) +servers = containers.sort_by{ |c| c["Name"] }.map do |c| + port = c["Publishers"][0]["PublishedPort"] + "localhost:#{port}" +end client = Dalli::Client.new(servers, serializer: JSON) diff --git a/test/support/test_client.ex b/test/support/test_client.ex index 5171ab4..d486ae6 100644 --- a/test/support/test_client.ex +++ b/test/support/test_client.ex @@ -1,7 +1,3 @@ defmodule TestClient do - use Cream.Client, servers: [ - "localhost:11201", - "localhost:11202", - "localhost:11203" - ] + use Cream.Client end From 34a49db146de2f45f39dbbbda8c6c39fd73f4d2d Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 13 Aug 2022 14:23:05 -0500 Subject: [PATCH 17/28] Simplify startup --- config/runtime.exs | 34 +++++++++------------------------- mix.exs | 2 -- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 135eb28..d424a6e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1,32 +1,16 @@ import Config -{:ok, _apps} = Application.ensure_all_started(:finch) -{:ok, _pid} = Finch.start_link(name: Finch) +{json, 0} = System.cmd("docker", ~w(compose ps --format json)) -{:ok, resp} = Finch.build(:get, "http://localhost/containers/json", [], nil, unix_socket: "/var/run/docker.sock") -|> Finch.request(Finch) +servers = Jason.decode!(json) +|> Enum.sort_by(& &1["Name"]) +|> Enum.map(fn container -> + port = container["Publishers"] + |> List.first() + |> Map.get("PublishedPort") -servers = Jason.decode!(resp.body) -|> Enum.reduce([], fn container, acc -> - name = Enum.find_value(container["Names"], fn name -> - if String.starts_with?(name, "/cream-ex-memcached-") do - name - else - false - end - end) - - if name do - [%{"PublicPort" => port}] = container["Ports"] - [{name, port} | acc] - else - acc - end + "localhost:#{port}" end) -|> Enum.sort() -|> Enum.map(fn {_name, port} -> "localhost:#{port}" end) -config :cream, Cream.Connection, server: Enum.at(servers, 0) +config :cream, Cream.Connection, server: List.first(servers) config :cream, Cream.Client, servers: servers - -:ok = NimblePool.stop(Finch) diff --git a/mix.exs b/mix.exs index 843e2d1..d0eda5f 100644 --- a/mix.exs +++ b/mix.exs @@ -64,8 +64,6 @@ defmodule Cream.Mixfile do {:connection, "~> 1.0"}, {:nimble_pool, "~> 0.0"}, {:jason, "~> 1.0", only: [:dev, :test]}, - # {:kestrel, "~> 0.0", git: "https://github.com/cjbottaro/kestrel_ex", only: [:dev, :test]}, - {:kestrel, "~> 0.0", path: "~/Projects/kestrel_ex", only: [:dev, :test]}, {:ex_doc, "~> 0.0", only: :dev}, ] end From a93c4c51b066fb819f8f1afdcd4119eeec20ff18 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 13 Aug 2022 14:30:05 -0500 Subject: [PATCH 18/28] Better docker compose file --- docker-compose.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 579ccf6..b23b6fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,20 +2,10 @@ version: "3" services: - memcached-1: - container_name: cream-ex-memcached-1 - image: memcached:alpine - ports: - - 11211 - - memcached-2: - container_name: cream-ex-memcached-2 - image: memcached:alpine - ports: - - 11211 - - memcached-3: - container_name: cream-ex-memcached-3 + memcached: image: memcached:alpine ports: - 11211 + deploy: + mode: replicated + replicas: 3 From 1d50da6960b5502e2ec7eb8346f6c34a902afdb8 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 14 Aug 2022 13:31:48 -0500 Subject: [PATCH 19/28] Use Cream.Item pervasively --- lib/cream/coder.ex | 47 ++++++++++++--- lib/cream/connection.ex | 127 +++++++++++++++------------------------ lib/cream/item.ex | 21 ++++++- lib/cream/protocol.ex | 2 +- test/client_test.exs | 2 +- test/coder_test.exs | 43 +++++++------ test/connection_test.exs | 22 +++---- 7 files changed, 142 insertions(+), 122 deletions(-) diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index 502c17b..ea69695 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -99,15 +99,26 @@ defmodule Cream.Coder do """ @callback decode(value, flags) :: {:ok, value} | {:error, reason} + def encode_item(%Cream.Item{} = item, conn, coder) do + with {:ok, coder} <- fetch_coder(conn, coder), + {:ok, value, flags} <- encode_value(coder, item.value, item.flags) + do + {:ok, %{item | raw_value: value, flags: flags, coder: coder}} + else + :not_found -> {:ok, item} + error -> error + end + end + @doc false - def encode(coder, value, flags) when is_atom(coder) do + def encode_value(coder, value, flags) when is_atom(coder) do coder.encode(value, flags) end @doc false - def encode(coders, value, flags) when is_list(coders) do + def encode_value(coders, value, flags) when is_list(coders) do Enum.reduce_while(coders, {:ok, value, flags}, fn coder, {:ok, value, flags} -> - case encode(coder, value, flags) do + case encode_value(coder, value, flags) do {:ok, value, flags} -> {:cont, {:ok, value, flags}} {:error, reason} -> {:halt, {:error, reason}} end @@ -115,18 +126,40 @@ defmodule Cream.Coder do end @doc false - def decode(coder, value, flags) when is_atom(coder) do + def decode_item(%Cream.Item{} = item, conn, coder) do + with {:ok, coder} <- fetch_coder(conn, coder), + {:ok, value} <- decode_value(coder, item.raw_value, item.flags) + do + {:ok, %{item | value: value, coder: coder}} + else + :not_found -> {:ok, item} + error -> error + end + end + + @doc false + def decode_value(coder, value, flags) when is_atom(coder) do coder.decode(value, flags) end @doc false - def decode(coders, value, flags) when is_list(coders) do - Enum.reduce_while(coders, {:ok, value}, fn coder, {:ok, value} -> - case decode(coder, value, flags) do + def decode_value(coders, value, flags) when is_list(coders) do + Enum.reverse(coders) + |> Enum.reduce_while({:ok, value}, fn coder, {:ok, value} -> + case decode_value(coder, value, flags) do {:ok, value} -> {:cont, {:ok, value}} {:error, reason} -> {:halt, {:error, reason}} end end) end + defp fetch_coder(_conn, false), do: :not_found + defp fetch_coder(_conn, coder) when coder != nil, do: {:ok, coder} + defp fetch_coder(conn, _coder) do + case Connection.call(conn, :fetch_coder) do + {:ok, nil} -> :not_found + result -> result + end + end + end diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 4b37f18..ad0f3f7 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -40,7 +40,7 @@ defmodule Cream.Connection do use Connection require Logger - alias Cream.{Protocol, Coder, Error, ConnectionError} + alias Cream.{Protocol, Item, Coder, Error, ConnectionError} @typedoc """ A connection. @@ -165,6 +165,8 @@ defmodule Cream.Connection do @doc """ Delete a key. + * `:quiet` - `(boolean)` - If `true`, ignore semantic errors. + iex> delete(conn, "foo") :ok @@ -174,12 +176,14 @@ defmodule Cream.Connection do """ @spec delete(t, binary, Keyword.t) :: :ok | {:error, Error.t} | {:error, ConnectionError.t} def delete(conn, key, opts \\ []) do + quiet? = Keyword.get(opts, :quiet, true) + case Connection.call(conn, {:delete, key}) do {:ok, %{status: :ok}} -> :ok - {:ok, %{status: :not_found}} -> if opts[:verbose] do - {:error, Error.exception(:not_found)} - else + {:ok, %{status: :not_found}} -> if quiet? do :ok + else + {:error, Error.exception(:not_found)} end {:error, %ConnectionError{} = conn_error} -> {:error, conn_error} {:error, reason} -> {:error, Error.exception(reason)} @@ -191,7 +195,7 @@ defmodule Cream.Connection do ## Options - * `:cas` - `(boolean)` - Return cas value in result. Default `false`. + * `:verbose` - `(boolean)` - If `true`, return the `t:Cream.Item.t/0` that was set. Default `false`. * `:coder` - `(atom|nil)` - Use a `Cream.Coder` on value. Overrides the config used by `start_link/1`. Default `nil`. @@ -212,32 +216,26 @@ defmodule Cream.Connection do :ok # Return cas value. - iex> set(conn, {"foo", "bar"}, cas: true) - {:ok, 123} + iex> set(conn, {"foo", "bar"}, verbose: true) + {:ok, %Cream.Item{cas: 123, ...}} # Set using bad cas value results in error. iex> set(conn, {"foo", "bar", cas: 124}) {:error, %Cream.Error{reason: :exists}} # Set using cas value and return new cas value. - iex> set(conn, {"foo", "bar", cas: 123}, cas: true) - {:ok, 124} + iex> set(conn, {"foo", "bar", cas: 123}, verbose: true) + {:ok, %Cream.Item{cas: 124, ...}} """ - @spec set(t, item, Keyword.t) :: :ok | {:ok, cas} | {:error, Error.t} | {:error, ConnectionError.t} + @spec set(t, item, Keyword.t) :: :ok | {:ok, Item.t} | {:error, Error.t} | {:error, ConnectionError.t} def set(conn, item, opts \\ []) do - opts_coder = case opts[:coder] do - false -> false - coder -> List.wrap(coder) - end - with {:ok, item} <- Cream.Item.from_args(item), - {:ok, conn_coder} <- Connection.call(conn, :get_encoder), - {:ok, item} <- encode(item, opts_coder, conn_coder), + {:ok, item} <- Coder.encode_item(item, conn, opts[:coder]), {:ok, %{status: :ok} = packet} <- Connection.call(conn, {:set, item}) do - if opts[:cas] do - {:ok, packet.cas} + if opts[:verbose] do + {:ok, %{item | cas: packet.cas}} else :ok end @@ -253,47 +251,52 @@ defmodule Cream.Connection do ## Options - * `:verbose` - `(boolean)` - If `true`, missing keys will return an error. If - `false`, missing keys will return `nil`. Default `false`. - * `:cas` - `(boolean)` - Return cas value in result. Default `false`. + * `:quiet` - `(boolean)` - If `false`, missing keys will return an error. Default `true`. + * `:verbose` - `(boolean)` - If `true`, `t:Cream.Item.t/0`s are returned. If false, + then just value is returned. Default `false`. ## Examples - iex> get(conn, "foo") - {:ok, nil} + iex> delete(conn, "name") + :ok - iex> get(conn, "foo", verbose: true) + iex> get(conn, "name", quiet: false) {:error, %Cream.Error{reason: :not_found}} + iex> get(conn, "name") + {:ok, nil} + + iex> set(conn, {"name", "Callie"}) + :ok + iex> get(conn, "name") {:ok, "Callie"} - iex> get(conn, "name", cas: true) - {:ok, "Callie", 123} + iex> get(conn, "name", verbose: true) + {:ok, %Cream.Item{value: "Callie", cas: 123, ...}} """ - @spec get(t, binary, Keyword.t) :: {:ok, term} | {:ok, term, cas} | {:error, Error.t} | {:error, ConnectionError.t} + @spec get(t, binary, Keyword.t) :: {:ok, term} | {:ok, Item.t} | {:error, Error.t} | {:error, ConnectionError.t} def get(conn, key, opts \\ []) do - opts_coder = case opts[:coder] do - false -> false - coder -> List.wrap(coder) |> Enum.reverse() - end + quiet? = Keyword.get(opts, :quiet, true) + verbose? = Keyword.get(opts, :verbose, false) - with {:ok, %{status: :ok} = packet, conn_coder} <- Connection.call(conn, {:get, key}), - {:ok, value} <- decode(packet, opts_coder, conn_coder) + with {:ok, %{status: :ok} = packet} <- Connection.call(conn, {:get, key}), + {:ok, item} <- Item.from_packet(packet, key: key), + {:ok, item} <- Coder.decode_item(item, conn, opts[:coder]) do - if opts[:cas] do - {:ok, value, packet.cas} + if verbose? do + {:ok, %{item | key: key}} else - {:ok, value} + {:ok, item.value} end else - {:ok, %{status: :not_found}, _coder} -> if opts[:verbose] do - {:error, Error.exception(:not_found)} - else + {:ok, %{status: :not_found}} -> if quiet? do {:ok, nil} + else + {:error, Error.exception(:not_found)} end - {:ok, %{status: reason}, _coder} -> {:error, Error.exception(reason)} + {:ok, %{status: reason}} -> {:error, Error.exception(reason)} {:error, %ConnectionError{} = conn_error} -> {:error, conn_error} {:error, reason} -> {:error, Error.exception(reason)} end @@ -315,14 +318,13 @@ defmodule Cream.Connection do """ @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:ok, term, cas} | {:error, Error.t} | {:error, ConnectionError.t} def fetch(conn, key, opts \\ [], f) do - case get(conn, key, Keyword.put(opts, :verbose, true)) do + case get(conn, key, Keyword.put(opts, :quiet, false)) do {:ok, value} -> {:ok, value} - {:ok, value, cas} -> {:ok, value, cas} {:error, %Error{reason: :not_found}} -> value = f.() case set(conn, {key, value}, opts) do :ok -> {:ok, value} - {:ok, cas} -> {:ok, value, cas} + {:ok, item} -> {:ok, item} error -> error end @@ -355,14 +357,9 @@ defmodule Cream.Connection do end def init(config) do - encoder = List.wrap(config[:coder]) - decoder = Enum.reverse(encoder) - state = %{ config: config, socket: nil, - encoder: encoder, - decoder: decoder, err_reason: nil, err_count: 0, } @@ -414,8 +411,8 @@ defmodule Cream.Connection do {:connect, reason, %{state | socket: nil}} end - def handle_call(:get_encoder, _from, state) do - {:reply, {:ok, state.encoder}, state} + def handle_call(:fetch_coder, _from, state) do + {:reply, {:ok, state.config[:coder]}, state} end def handle_call(_command, _from, state) when is_nil(state.socket) do @@ -468,7 +465,7 @@ defmodule Cream.Connection do end def handle_call({:get, key}, from, state) do - %{socket: socket, decoder: decoder} = state + %{socket: socket} = state packet = Protocol.get(key) @@ -477,7 +474,7 @@ defmodule Cream.Connection do {:ok, packet} <- Protocol.recv_packet(socket), :ok <- :inet.setopts(socket, active: true) do - {:reply, {:ok, packet, decoder}, state} + {:reply, {:ok, packet}, state} else {:error, reason} -> handle_conn_error(reason, from, state) end @@ -507,28 +504,4 @@ defmodule Cream.Connection do {:disconnect, reason, state} end - defp encode(%Cream.Item{} = item, opts_coder, conn_coder) do - result = case {opts_coder, conn_coder} do - {false, _} -> :noop - {[], []} -> :noop - {coder, _} when coder != [] -> Coder.encode(coder, item.value, item.flags) - {_, coder} when coder != [] -> Coder.encode(coder, item.value, item.flags) - end - - case result do - :noop -> {:ok, item} - {:ok, value, flags} -> {:ok, %{item | value: value, flags: flags}} - error -> error - end - end - - defp decode(packet, opts_coder, conn_coder) do - case {opts_coder, conn_coder} do - {false, _} -> {:ok, packet.value} - {[], []} -> {:ok, packet.value} - {coder, _} when coder != [] -> Coder.decode(coder, packet.value, packet.extras.flags) - {_, coder} when coder != [] -> Coder.decode(coder, packet.value, packet.extras.flags) - end - end - end diff --git a/lib/cream/item.ex b/lib/cream/item.ex index 76dd57d..08a2a12 100644 --- a/lib/cream/item.ex +++ b/lib/cream/item.ex @@ -1,11 +1,26 @@ defmodule Cream.Item do - defstruct [:key, :value, ttl: 0, cas: 0, flags: 0] + defstruct [:key, :value, :raw_value, ttl: 0, cas: 0, flags: 0, coder: nil] def from_args({key, value}) do - {:ok, %__MODULE__{key: key, value: value}} + {:ok, %__MODULE__{key: key, value: value, raw_value: value}} end def from_args({key, value, opts}) do - {:ok, struct!(__MODULE__, Keyword.merge(opts, [key: key, value: value]))} + with {:ok, item} <- from_args({key, value}) do + {:ok, struct!(item, opts)} + end + end + + def from_packet(packet, fields \\ []) do + item = struct(__MODULE__, [ + key: packet.key || fields[:key], + value: packet.value, + raw_value: packet.value, + ttl: :unknown, + cas: packet.cas, + flags: packet.extras.flags + ]) + + {:ok, item} end end diff --git a/lib/cream/protocol.ex b/lib/cream/protocol.ex index c15d701..feb12cc 100644 --- a/lib/cream/protocol.ex +++ b/lib/cream/protocol.ex @@ -90,7 +90,7 @@ defmodule Cream.Protocol do end def set(item) do - %{key: key, value: value, ttl: ttl, cas: cas, flags: flags} = item + %{key: key, raw_value: value, ttl: ttl, cas: cas, flags: flags} = item key_size = byte_size(key) value_size = byte_size(value) diff --git a/test/client_test.exs b/test/client_test.exs index 6de1ec6..7d8a3a3 100644 --- a/test/client_test.exs +++ b/test/client_test.exs @@ -39,7 +39,7 @@ defmodule ClientTest do test "fetch" do :ok = TestClient.delete("foo") - {:error, %Error{reason: :not_found}} = TestClient.get("foo", verbose: true) + {:error, %Error{reason: :not_found}} = TestClient.get("foo", quiet: false) {:ok, "bar"} = TestClient.fetch("foo", fn -> "bar" end) {:ok, "bar"} = TestClient.get("foo") diff --git a/test/coder_test.exs b/test/coder_test.exs index 3429958..b21170a 100644 --- a/test/coder_test.exs +++ b/test/coder_test.exs @@ -4,54 +4,53 @@ defmodule CoderTest do alias Cream.Coder test "jason" do - {:ok, json, 0b1} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0) + {:ok, json, 0b1} = Coder.encode_value(Coder.Jason, %{"foo" => "bar"}, 0) {:ok, %{"foo" => "bar"}} = Jason.decode(json) - {:ok, json, 0b101} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0b100) + {:ok, json, 0b101} = Coder.encode_value(Coder.Jason, %{"foo" => "bar"}, 0b100) {:ok, %{"foo" => "bar"}} = Jason.decode(json) - {:error, _reason} = Coder.encode(Coder.Jason, {"foo", "bar"}, 0b100) + {:error, _reason} = Coder.encode_value(Coder.Jason, {"foo", "bar"}, 0b100) - {:ok, json, flags} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0) - {:ok, %{"foo" => "bar"}} = Coder.decode(Coder.Jason, json, flags) + {:ok, json, flags} = Coder.encode_value(Coder.Jason, %{"foo" => "bar"}, 0) + {:ok, %{"foo" => "bar"}} = Coder.decode_value(Coder.Jason, json, flags) - {:ok, json, flags} = Coder.encode(Coder.Jason, %{"foo" => "bar"}, 0b100) - {:ok, %{"foo" => "bar"}} = Coder.decode(Coder.Jason, json, flags) + {:ok, json, flags} = Coder.encode_value(Coder.Jason, %{"foo" => "bar"}, 0b100) + {:ok, %{"foo" => "bar"}} = Coder.decode_value(Coder.Jason, json, flags) end test "gzip" do - {:ok, zipped, 0b10} = Coder.encode(Coder.Gzip, "foobar", 0) + {:ok, zipped, 0b10} = Coder.encode_value(Coder.Gzip, "foobar", 0) "foobar" = :zlib.gunzip(zipped) - {:ok, zipped, 0b110} = Coder.encode(Coder.Gzip, "foobar", 0b100) + {:ok, zipped, 0b110} = Coder.encode_value(Coder.Gzip, "foobar", 0b100) "foobar" = :zlib.gunzip(zipped) - {:error, _reason} = Coder.encode(Coder.Gzip, :foobar, 0) + {:error, _reason} = Coder.encode_value(Coder.Gzip, :foobar, 0) - {:ok, zipped, flags} = Coder.encode(Coder.Gzip, "foobar", 0) - {:ok, "foobar"} = Coder.decode(Coder.Gzip, zipped, flags) + {:ok, zipped, flags} = Coder.encode_value(Coder.Gzip, "foobar", 0) + {:ok, "foobar"} = Coder.decode_value(Coder.Gzip, zipped, flags) - {:ok, zipped, flags} = Coder.encode(Coder.Gzip, "foobar", 0b100) - {:ok, "foobar"} = Coder.decode(Coder.Gzip, zipped, flags) + {:ok, zipped, flags} = Coder.encode_value(Coder.Gzip, "foobar", 0b100) + {:ok, "foobar"} = Coder.decode_value(Coder.Gzip, zipped, flags) end test "jason + gzip" do - encoders = [Coder.Jason, Coder.Gzip] - decoders = [Coder.Gzip, Coder.Jason] + coders = [Coder.Jason, Coder.Gzip] term = %{"foo" => "bar"} - {:ok, data, 0b11} = Coder.encode(encoders, term, 0) + {:ok, data, 0b11} = Coder.encode_value(coders, term, 0) ^term = data |> :zlib.gunzip() |> Jason.decode!() - {:ok, data, 0b111} = Coder.encode(encoders, term, 0b100) + {:ok, data, 0b111} = Coder.encode_value(coders, term, 0b100) ^term = data |> :zlib.gunzip() |> Jason.decode!() - {:ok, data, 0b11} = Coder.encode(encoders, term, 0) - {:ok, ^term} = Coder.decode(decoders, data, 0b11) + {:ok, data, 0b11} = Coder.encode_value(coders, term, 0) + {:ok, ^term} = Coder.decode_value(coders, data, 0b11) - {:ok, data, flags} = Coder.encode(encoders, term, 0b100) - {:ok, ^term} = Coder.decode(decoders, data, flags) + {:ok, data, flags} = Coder.encode_value(coders, term, 0b100) + {:ok, ^term} = Coder.decode_value(coders, data, flags) end end diff --git a/test/connection_test.exs b/test/connection_test.exs index bef9a06..817ae7b 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -15,7 +15,7 @@ defmodule ConnectionTest do test "set", %{conn: conn} do :ok = Connection.set(conn, {"foo", "bar"}) - {:ok, cas} = Connection.set(conn, {"foo", "bar"}, cas: true) + {:ok, %{cas: cas}} = Connection.set(conn, {"foo", "bar"}, verbose: true) {:error, %Error{reason: :exists}} = Connection.set(conn, {"foo", "bar1", cas: cas+1}) {:ok, "bar"} = Connection.get(conn, "foo") :ok = Connection.set(conn, {"foo", "bar1", cas: cas}) @@ -23,34 +23,34 @@ defmodule ConnectionTest do end test "get", %{conn: conn} do - {:ok, set_cas} = Connection.set(conn, {"name", "Callie"}, cas: true) + {:ok, %{cas: set_cas}} = Connection.set(conn, {"name", "Callie"}, verbose: true) {:ok, nil} = Connection.get(conn, "foo") - {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", quiet: false) {:ok, "Callie"} = Connection.get(conn, "name") - {:ok, "Callie", get_cas} = Connection.get(conn, "name", cas: true) + {:ok, %{value: "Callie", cas: get_cas}} = Connection.get(conn, "name", verbose: true) assert set_cas == get_cas end test "delete", %{conn: conn} do - {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", quiet: false) :ok = Connection.delete(conn, "foo") :ok = Connection.set(conn, {"foo", "bar"}) {:ok, "bar"} = Connection.get(conn, "foo") - :ok = Connection.delete(conn, "foo", verbose: true) - {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) - {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", verbose: true) + :ok = Connection.delete(conn, "foo", quiet: false) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", quiet: false) + {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", quiet: false) :ok = Connection.set(conn, {"foo", "bar"}) {:ok, "bar"} = Connection.get(conn, "foo") :ok = Connection.delete(conn, "foo") - {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) - {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", quiet: false) + {:error, %Error{reason: :not_found}} = Connection.delete(conn, "foo", quiet: false) end test "fetch", %{conn: conn} do - {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", verbose: true) + {:error, %Error{reason: :not_found}} = Connection.get(conn, "foo", quiet: false) {:ok, "bar"} = Connection.fetch(conn, "foo", fn -> "bar" end) {:ok, "bar"} = Connection.get(conn, "foo") From 81a5c5b69619e6bd2ae6ef1eab590a464efc28cd Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 14 Aug 2022 13:40:47 -0500 Subject: [PATCH 20/28] Specs are good --- lib/cream/coder.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index ea69695..fd39536 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -85,6 +85,7 @@ defmodule Cream.Coder do order. """ + @type t :: module @type value :: term @type flags :: integer() @type reason :: term @@ -99,6 +100,8 @@ defmodule Cream.Coder do """ @callback decode(value, flags) :: {:ok, value} | {:error, reason} + @doc false + @spec encode_item(Cream.Item.t, Cream.Connection.t, Cream.Coder.t | [Cream.Coder.t]) :: {:ok, Cream.Item.t} | {:error, reason} def encode_item(%Cream.Item{} = item, conn, coder) do with {:ok, coder} <- fetch_coder(conn, coder), {:ok, value, flags} <- encode_value(coder, item.value, item.flags) @@ -111,11 +114,13 @@ defmodule Cream.Coder do end @doc false + @spec encode_value(Cream.Coder.t, term, non_neg_integer) :: {:ok, term, non_neg_integer} | {:error, reason} def encode_value(coder, value, flags) when is_atom(coder) do coder.encode(value, flags) end @doc false + @spec encode_value([Cream.Coder.t], term, non_neg_integer) :: {:ok, term, non_neg_integer} | {:error, reason} def encode_value(coders, value, flags) when is_list(coders) do Enum.reduce_while(coders, {:ok, value, flags}, fn coder, {:ok, value, flags} -> case encode_value(coder, value, flags) do @@ -126,6 +131,7 @@ defmodule Cream.Coder do end @doc false + @spec decode_item(Cream.Item.t, Cream.Connection.t, Cream.Coder.t | [Cream.Coder.t]) :: {:ok, Cream.Item.t} | {:error, reason} def decode_item(%Cream.Item{} = item, conn, coder) do with {:ok, coder} <- fetch_coder(conn, coder), {:ok, value} <- decode_value(coder, item.raw_value, item.flags) @@ -138,11 +144,13 @@ defmodule Cream.Coder do end @doc false + @spec decode_value(Cream.Coder.t, term, non_neg_integer) :: {:ok, term} | {:error, reason} def decode_value(coder, value, flags) when is_atom(coder) do coder.decode(value, flags) end @doc false + @spec decode_value([Cream.Coder.t], term, non_neg_integer) :: {:ok, term} | {:error, reason} def decode_value(coders, value, flags) when is_list(coders) do Enum.reverse(coders) |> Enum.reduce_while({:ok, value}, fn coder, {:ok, value} -> @@ -153,6 +161,7 @@ defmodule Cream.Coder do end) end + @spec fetch_coder(Cream.Connection.t, Cream.Coder.t | false | nil) :: {:ok, t} | :not_found | {:error, reason} defp fetch_coder(_conn, false), do: :not_found defp fetch_coder(_conn, coder) when coder != nil, do: {:ok, coder} defp fetch_coder(conn, _coder) do From 2023b38150e3e1422e9d1a796f7c291ce4071a7e Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 14 Aug 2022 13:58:53 -0500 Subject: [PATCH 21/28] Fix flush --- lib/cream/cluster.ex | 15 ++++----------- lib/cream/coder.ex | 2 +- lib/cream/connection.ex | 2 +- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/cream/cluster.ex b/lib/cream/cluster.ex index f015d5b..fb8d57c 100644 --- a/lib/cream/cluster.ex +++ b/lib/cream/cluster.ex @@ -48,21 +48,14 @@ defmodule Cream.Cluster do end def flush(cluster, opts \\ []) do - errors = cluster.servers - |> Enum.with_index() - |> Enum.reduce(%{}, fn {server, i}, acc -> + (0..tuple_size(cluster.connections)-1) + |> Enum.reduce_while(:ok, fn i, :ok -> conn = elem(cluster.connections, i) case Cream.Connection.flush(conn, opts) do - :ok -> acc - {:error, reason} -> Map.put(acc, server, reason) + :ok -> {:cont, :ok} + error -> {:halt, error} end end) - - if errors == %{} do - :ok - else - {:error, errors} - end end def find_conn(%{continuum: nil, connections: {conn}}, _key), do: conn diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index fd39536..2559c8c 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -87,7 +87,7 @@ defmodule Cream.Coder do @type t :: module @type value :: term - @type flags :: integer() + @type flags :: non_neg_integer @type reason :: term @doc """ diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index ad0f3f7..7f20b0c 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -158,7 +158,7 @@ defmodule Cream.Connection do :ok -> :ok reason -> {:error, Error.exception(reason)} end - %ConnectionError{} = conn_error -> {:error, conn_error} + error -> error end end From ec32a6e0886f110ac8ca823f8c7fe74f92292690 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 14 Aug 2022 14:33:07 -0500 Subject: [PATCH 22/28] Read through cache --- lib/cream/connection.ex | 52 ++++++++++++++++++++++++++++++++-------- test/client_test.exs | 2 +- test/connection_test.exs | 9 ++++++- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 7f20b0c..d31c2e5 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -303,20 +303,50 @@ defmodule Cream.Connection do end @doc """ - Get a key and set its value if it doesn't exist. + Read through cache. - If `key` doesn't exist, then `f` is used to generate its value and set it on the server. + If `key` doesn't exist, then `f` is used to generate its value and set it on + the server. - iex> :ok = flush(conn) - iex> {:ok, nil} = get(conn, "foo") - iex> {:ok, "bar"} = fetch(conn, "foo", fn -> "bar" end) - iex> {:ok, "bar"} = get(conn, "foo") + It is similar to the following code: - `set/3` options will be used if the key doesn't exist. + case get(conn, key, quiet: false) do + {:ok, value} -> {:ok, value} + {:error, %Cream.Error{reason: :not_found}} -> + value = generate_value() + set(conn, {key, value}) + {:ok, value} + end + + Note that this function never returns `t:Cream.ConnectionError.t/0`. If the + memcached server is down, it simply invokes `f` and returns that value. + + It also never returns `t:Cream.Item.t/0` since that _requires_ talking to the + memcached server. + + In other words, this function is meant to always succeed, even if your memcached + servers are down. + + ## Examples + + # Key does not exist + iex> get(conn, "foo") + {:ok, nil} + + # Cache miss + iex> fetch(conn, "foo", fn -> "bar" end) + {:ok, "bar"} + + # Cache has been populated (key exists now) + iex> get(conn, "foo") + {:ok, "bar"} + + # Cache hit + iex> fetch(conn, "foo", fn -> "rab" end) + {:ok, "bar"} - `get/3` options will always be used. """ - @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:ok, term, cas} | {:error, Error.t} | {:error, ConnectionError.t} + @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:error, Error.t} def fetch(conn, key, opts \\ [], f) do case get(conn, key, Keyword.put(opts, :quiet, false)) do {:ok, value} -> {:ok, value} @@ -324,10 +354,12 @@ defmodule Cream.Connection do value = f.() case set(conn, {key, value}, opts) do :ok -> {:ok, value} - {:ok, item} -> {:ok, item} + {:ok, _item} -> {:ok, value} + {:error, %ConnectionError{}} -> {:ok, value} error -> error end + {:error, %ConnectionError{}} -> {:ok, f.()} error -> error end end diff --git a/test/client_test.exs b/test/client_test.exs index 7d8a3a3..f2f6df3 100644 --- a/test/client_test.exs +++ b/test/client_test.exs @@ -1,6 +1,6 @@ defmodule ClientTest do use ExUnit.Case - alias Cream.Error + alias Cream.{Error} setup_all do {:ok, _client} = TestClient.start_link() diff --git a/test/connection_test.exs b/test/connection_test.exs index 817ae7b..56526eb 100644 --- a/test/connection_test.exs +++ b/test/connection_test.exs @@ -2,7 +2,7 @@ defmodule ConnectionTest do use ExUnit.Case # doctest Cream.Connection, import: true, only: [set: 3] - alias Cream.{Connection, Error, Coder} + alias Cream.{Connection, Error, ConnectionError, Coder} setup_all do {:ok, conn} = Connection.start_link() @@ -59,6 +59,13 @@ defmodule ConnectionTest do {:ok, "baz"} = Connection.get(conn, "foo") end + @tag capture_log: true + test "fetch with bad connection" do + {:ok, conn} = Connection.start_link(server: "foobar:22133") + {:error, %ConnectionError{}} = Connection.get(conn, "foo") + {:ok, "bar"} = Connection.fetch(conn, "foo", fn -> "bar" end) + end + # Note that this uses Cream.Coder.Json which is different from # Cream.Coder.Jason and only exists in test env. test "single coder", %{conn: conn} do From da1922c7eab277eb7f1305176631036b862b1cde Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 14 Aug 2022 14:34:27 -0500 Subject: [PATCH 23/28] More docs --- lib/cream/connection.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index d31c2e5..dbba913 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -345,6 +345,10 @@ defmodule Cream.Connection do iex> fetch(conn, "foo", fn -> "rab" end) {:ok, "bar"} + # Set using ttl on cache miss + iex> fetch(conn, "foo", [ttl: 60], fn -> "bar" end) + {:ok, "bar"} + """ @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:error, Error.t} def fetch(conn, key, opts \\ [], f) do From fc317b81770a0668b6f6af5d4bda2127da20f454 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sun, 14 Aug 2022 15:35:06 -0500 Subject: [PATCH 24/28] Clean up logging --- lib/cream/coder.ex | 37 +++++++++++++++++++--------- lib/cream/coder/gzip.ex | 5 ++++ lib/cream/coder/jason.ex | 5 ++++ lib/cream/connection.ex | 8 +++--- lib/cream/errors/connection_error.ex | 8 ++++++ lib/cream/errors/error.ex | 13 ++++++++++ lib/cream/errors/multi_error.ex | 6 ++++- lib/cream/item.ex | 31 +++++++++++++++++++++++ 8 files changed, 98 insertions(+), 15 deletions(-) diff --git a/lib/cream/coder.ex b/lib/cream/coder.ex index 2559c8c..84b96f3 100644 --- a/lib/cream/coder.ex +++ b/lib/cream/coder.ex @@ -85,23 +85,38 @@ defmodule Cream.Coder do order. """ + @typedoc """ + Any module that implements the `Cream.Coder` behaviour. + """ @type t :: module - @type value :: term + + @typedoc """ + Flags bitmask. + """ @type flags :: non_neg_integer - @type reason :: term + + @typedoc """ + Actual binary value stored in memcached. + """ + @type raw_value :: binary + + @typedoc """ + Value to be serialized, or value after deserialization. + """ + @type value :: term @doc """ Encode a value and set flags. """ - @callback encode(value, flags) :: {:ok, value, flags} | {:error, reason} + @callback encode(value, flags) :: {:ok, raw_value, flags} | {:error, term} @doc """ Decode a value based on flags. """ - @callback decode(value, flags) :: {:ok, value} | {:error, reason} + @callback decode(raw_value, flags) :: {:ok, value} | {:error, term} @doc false - @spec encode_item(Cream.Item.t, Cream.Connection.t, Cream.Coder.t | [Cream.Coder.t]) :: {:ok, Cream.Item.t} | {:error, reason} + @spec encode_item(Cream.Item.t, Cream.Connection.t, Cream.Coder.t | [Cream.Coder.t]) :: {:ok, Cream.Item.t} | {:error, term} def encode_item(%Cream.Item{} = item, conn, coder) do with {:ok, coder} <- fetch_coder(conn, coder), {:ok, value, flags} <- encode_value(coder, item.value, item.flags) @@ -114,13 +129,13 @@ defmodule Cream.Coder do end @doc false - @spec encode_value(Cream.Coder.t, term, non_neg_integer) :: {:ok, term, non_neg_integer} | {:error, reason} + @spec encode_value(Cream.Coder.t, term, non_neg_integer) :: {:ok, term, non_neg_integer} | {:error, term} def encode_value(coder, value, flags) when is_atom(coder) do coder.encode(value, flags) end @doc false - @spec encode_value([Cream.Coder.t], term, non_neg_integer) :: {:ok, term, non_neg_integer} | {:error, reason} + @spec encode_value([Cream.Coder.t], term, non_neg_integer) :: {:ok, term, non_neg_integer} | {:error, term} def encode_value(coders, value, flags) when is_list(coders) do Enum.reduce_while(coders, {:ok, value, flags}, fn coder, {:ok, value, flags} -> case encode_value(coder, value, flags) do @@ -131,7 +146,7 @@ defmodule Cream.Coder do end @doc false - @spec decode_item(Cream.Item.t, Cream.Connection.t, Cream.Coder.t | [Cream.Coder.t]) :: {:ok, Cream.Item.t} | {:error, reason} + @spec decode_item(Cream.Item.t, Cream.Connection.t, Cream.Coder.t | [Cream.Coder.t]) :: {:ok, Cream.Item.t} | {:error, term} def decode_item(%Cream.Item{} = item, conn, coder) do with {:ok, coder} <- fetch_coder(conn, coder), {:ok, value} <- decode_value(coder, item.raw_value, item.flags) @@ -144,13 +159,13 @@ defmodule Cream.Coder do end @doc false - @spec decode_value(Cream.Coder.t, term, non_neg_integer) :: {:ok, term} | {:error, reason} + @spec decode_value(Cream.Coder.t, term, non_neg_integer) :: {:ok, term} | {:error, term} def decode_value(coder, value, flags) when is_atom(coder) do coder.decode(value, flags) end @doc false - @spec decode_value([Cream.Coder.t], term, non_neg_integer) :: {:ok, term} | {:error, reason} + @spec decode_value([Cream.Coder.t], term, non_neg_integer) :: {:ok, term} | {:error, term} def decode_value(coders, value, flags) when is_list(coders) do Enum.reverse(coders) |> Enum.reduce_while({:ok, value}, fn coder, {:ok, value} -> @@ -161,7 +176,7 @@ defmodule Cream.Coder do end) end - @spec fetch_coder(Cream.Connection.t, Cream.Coder.t | false | nil) :: {:ok, t} | :not_found | {:error, reason} + @spec fetch_coder(Cream.Connection.t, Cream.Coder.t | false | nil) :: {:ok, t} | :not_found | {:error, term} defp fetch_coder(_conn, false), do: :not_found defp fetch_coder(_conn, coder) when coder != nil, do: {:ok, coder} defp fetch_coder(conn, _coder) do diff --git a/lib/cream/coder/gzip.ex b/lib/cream/coder/gzip.ex index 7426a63..ec99664 100644 --- a/lib/cream/coder/gzip.ex +++ b/lib/cream/coder/gzip.ex @@ -1,4 +1,9 @@ defmodule Cream.Coder.Gzip do + @moduledoc """ + A compression coder using `:zlib.g(un)zip`. + + It's uses the second bit on flags. + """ use Bitwise @behaviour Cream.Coder diff --git a/lib/cream/coder/jason.ex b/lib/cream/coder/jason.ex index 22899de..cbde2cc 100644 --- a/lib/cream/coder/jason.ex +++ b/lib/cream/coder/jason.ex @@ -1,4 +1,9 @@ defmodule Cream.Coder.Jason do + @moduledoc """ + A JSON codering using `Jason`. + + Uses the first bit on flags. + """ use Bitwise @behaviour Cream.Coder diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index dbba913..a1995a1 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -7,7 +7,7 @@ defmodule Cream.Connection do ## Global configuration - You can globally configure _all connections_ via `Config`. + You can globally configure _all connections_. ``` import Config @@ -28,7 +28,7 @@ defmodule Cream.Connection do 1. (Re)connections are retried indefinitely. While a connection is in a disconnected state, any operations on the - connection will result in `{:error, :not_connected}`. + connection will result in `{:error, %Cream.ConnectionError{...}}`. ## Value serialization @@ -167,6 +167,8 @@ defmodule Cream.Connection do * `:quiet` - `(boolean)` - If `true`, ignore semantic errors. + ## Examples + iex> delete(conn, "foo") :ok @@ -321,7 +323,7 @@ defmodule Cream.Connection do Note that this function never returns `t:Cream.ConnectionError.t/0`. If the memcached server is down, it simply invokes `f` and returns that value. - It also never returns `t:Cream.Item.t/0` since that _requires_ talking to the + It also never returns `t:Cream.Item.t/0` since that requires talking to the memcached server. In other words, this function is meant to always succeed, even if your memcached diff --git a/lib/cream/errors/connection_error.ex b/lib/cream/errors/connection_error.ex index 14f7e81..901423f 100644 --- a/lib/cream/errors/connection_error.ex +++ b/lib/cream/errors/connection_error.ex @@ -1,4 +1,12 @@ defmodule Cream.ConnectionError do + @moduledoc """ + Server connection errors. + + `reason` will be things like `:nxdomain` and `:econnrefused` as returned by + Erlang's `:gen_tcp` module. + + `server` will be a server binary like `"localhost:11211"`. + """ defexception [:reason, :server] @type t :: %__MODULE__{reason: atom, server: binary} diff --git a/lib/cream/errors/error.ex b/lib/cream/errors/error.ex index c8d15cf..44edebd 100644 --- a/lib/cream/errors/error.ex +++ b/lib/cream/errors/error.ex @@ -1,4 +1,17 @@ defmodule Cream.Error do + @moduledoc """ + Semantic errors returned by memcached server. + + The `reason` field corresponds to a packet's status as described by + the memcached binary protocol. + + Notable reasons are... + + * `not_found` - Key does not exist. + * `exists` - Usually a cas error. + + """ + defexception [:reason] @type t :: %__MODULE__{reason: atom} diff --git a/lib/cream/errors/multi_error.ex b/lib/cream/errors/multi_error.ex index ad43533..331b6ce 100644 --- a/lib/cream/errors/multi_error.ex +++ b/lib/cream/errors/multi_error.ex @@ -1,3 +1,7 @@ +# Maybe needed if we ever implement mset/mget, but maybe not. +# Instead of MultiError, we could just reduce_while until we +# hit the first error and just return that. defmodule Cream.MultiError do - defexception [:keys] + @moduledoc false + defexception [:message] end diff --git a/lib/cream/item.ex b/lib/cream/item.ex index 08a2a12..33b1d74 100644 --- a/lib/cream/item.ex +++ b/lib/cream/item.ex @@ -1,16 +1,47 @@ defmodule Cream.Item do + @moduledoc """ + Represents a conceptual cache item. + + This is returned by read and write type operations when the `verbose: true` + option is used. + + The memcached protocol doesn't give a way get the ttl of a key, so on reads + the `:ttl` field will be `:unknown`. + + `:raw_value` is the binary that is actually stored in memcached. + + `:value` is before (write) or after (read) `:coder` serialization has + been applied. It can be any kind of term. + + """ defstruct [:key, :value, :raw_value, ttl: 0, cas: 0, flags: 0, coder: nil] + @type t :: %__MODULE__{ + key: binary, + value: term, + raw_value: binary, + ttl: non_neg_integer, + cas: non_neg_integer, + flags: non_neg_integer, + coder: nil | Cream.Coder.t | [Cream.Coder.t] + } + + @doc false + @spec from_args({binary, term}) :: {:ok, t} def from_args({key, value}) do {:ok, %__MODULE__{key: key, value: value, raw_value: value}} end + @doc false + @spec from_args({binary, term, Keyword.t}) :: {:ok, t} def from_args({key, value, opts}) do with {:ok, item} <- from_args({key, value}) do {:ok, struct!(item, opts)} end end + @doc false + @spec from_packet(map, Keyword.t) :: {:ok, t} def from_packet(packet, fields \\ []) do item = struct(__MODULE__, [ key: packet.key || fields[:key], From 1cf11b45aaa556b47a2e7bf0d95839baf219856e Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 25 Mar 2023 14:33:08 -0500 Subject: [PATCH 25/28] Docs, specs, config --- Dockerfile | 5 ++ docker-compose.yml | 6 +++ lib/cream/application.ex | 4 +- lib/cream/client.ex | 78 +++++++++++++++------------- lib/cream/coder/jason.ex | 2 +- lib/cream/connection.ex | 24 ++++----- lib/cream/errors/connection_error.ex | 4 +- lib/cream/errors/error.ex | 6 +-- lib/cream/logger.ex | 1 + 9 files changed, 73 insertions(+), 57 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8327465 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM ruby:2.7-alpine + +ADD Gemfile test/support/populate.rb ./ +RUN bundle install +ENTRYPOINT [ "/bin/sleep" ] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b23b6fa..8ef74fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,3 +9,9 @@ services: deploy: mode: replicated replicas: 3 + + dalli: + tty: true + stdin_open: true + build: + context: . \ No newline at end of file diff --git a/lib/cream/application.ex b/lib/cream/application.ex index 1238776..7f8887d 100644 --- a/lib/cream/application.ex +++ b/lib/cream/application.ex @@ -6,7 +6,9 @@ defmodule Cream.Application do def start(_, _) do :ok = Cream.Logger.init() - children = [] + children = [ + # {Cream.Client, [name: Cream]} + ] Supervisor.start_link(children, strategy: :one_for_one) end diff --git a/lib/cream/client.ex b/lib/cream/client.ex index 7432078..c4b9e1b 100644 --- a/lib/cream/client.ex +++ b/lib/cream/client.ex @@ -25,7 +25,7 @@ defmodule Cream.Client do ``` Now _every_ client will use those servers unless overwritten by an argument - passed to `start_link/1` or `child_spec/1`. + passed to `start_link/1`. ## Using as a module @@ -63,20 +63,6 @@ defmodule Cream.Client do See `c:config/0` for an example. """ - @typedoc """ - A `Cream.Client`. - """ - @type t :: GenServer.server() - - @typedoc """ - Error reason. - """ - @type reason :: binary | atom | term - - alias Cream.{Cluster} - - @behaviour NimblePool - @defaults [ pool_size: 5, lazy: true, @@ -85,23 +71,44 @@ defmodule Cream.Client do ttl: nil ] - @doc """ - Default config. - - ``` - #{inspect @defaults, pretty: true, width: 0} - ``` + @typedoc """ + Configuration options. * `pool_size` - How big the connection pool is. * `lazy` - If the connection pool is lazily loaded. * `servers` - What memcached servers to connect to. * `coder` - What `Cream.Coder` to use. * `ttl` - Default time to live (expiry) in seconds to use with `set/3`. + + Defaults are: + ``` + #{inspect @defaults, pretty: true, width: 0} + ``` """ - def defaults, do: @defaults + @type config :: [ + {:pool_size, non_neg_integer}, + {:lazy, boolean}, + {:servers, [binary]}, + {:coder, module | nil}, + {:ttl, non_neg_integer | nil} + ] + + @typedoc """ + A `Cream.Client`. + """ + @type t :: GenServer.server() + + @typedoc """ + Error reason. + """ + @type reason :: binary | atom | term + + alias Cream.{Cluster} + + @behaviour NimblePool @doc """ - `Config` merged with `defaults/0`. + Configuration as read from `Config`. ``` import Config @@ -146,11 +153,13 @@ defmodule Cream.Client do `config` will be merged over `config/0`. """ def child_spec(config \\ []) do - config = Keyword.merge(config(), config) + {nimble_config, cream_config} = + config() + |> Keyword.merge(config) + |> Keyword.split([:pool_size, :lazy, :name]) - Keyword.take(config, [:pool_size, :lazy]) - |> Keyword.put(:worker, {__MODULE__, config}) - |> Keyword.put(:name, config[:name]) + nimble_config + |> Keyword.put(:worker, {__MODULE__, cream_config}) |> NimblePool.child_spec() end @@ -159,8 +168,12 @@ defmodule Cream.Client do `config` will be merged over `config/0`. - See `defaults/0` for valid config options. + Default options are: + ``` + #{inspect @defaults, pretty: true, width: 0} + ``` """ + @spec start_link(config) :: {:ok, t} | {:error, reason} def start_link(config \\ []) do %{start: {m, f, a}} = child_spec(config) apply(m, f, a) @@ -222,8 +235,6 @@ defmodule Cream.Client do Start a client. `config` is merged with `c:config/0`. - - See `defaults/0` for valid `config` options. """ @callback start_link(config :: Keyword.t) :: {:ok, t} | {:error, reason} @@ -231,8 +242,6 @@ defmodule Cream.Client do Child specification for supervisors. `config` is merged with `c:config/0`. - - See `defaults/0` for valid `config` options. """ @callback child_spec(config :: Keyword.t) :: Supervisor.child_spec @@ -281,11 +290,6 @@ defmodule Cream.Client do Cream.Client.flush(__MODULE__, opts) end - defp config_config do - Application.get_application(__MODULE__) - |> Application.get_env(__MODULE__, []) - end - end end diff --git a/lib/cream/coder/jason.ex b/lib/cream/coder/jason.ex index cbde2cc..13e111f 100644 --- a/lib/cream/coder/jason.ex +++ b/lib/cream/coder/jason.ex @@ -1,6 +1,6 @@ defmodule Cream.Coder.Jason do @moduledoc """ - A JSON codering using `Jason`. + A JSON coder using `Jason`. Uses the first bit on flags. """ diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index a1995a1..3452d3d 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -71,23 +71,18 @@ defmodule Cream.Connection do """ @type cas :: non_neg_integer() + @type config :: [ + {:server, binary}, + {:coder, module | nil} + ] + @defaults [ server: "localhost:11211", coder: nil, ] @doc """ - Default config. - - ``` - #{inspect @defaults, pretty: true, width: 0} - ``` - """ - def defaults, do: @defaults - - @doc """ - `Config` merged with `defaults/0`. - + Config options as read from `Config`. ``` import Config config :cream, Cream.Connection, coder: FooCoder @@ -108,7 +103,10 @@ defmodule Cream.Connection do The given `config` will be merged over `config/0`. - See `defaults/0` for config options. + Default config is + ``` + #{inspect @defaults, pretty: true, width: 0} + ``` Note that this will typically _always_ return `{:ok conn}`, even if the actual TCP connection failed. If that is the case, subsequent uses of the connection @@ -133,7 +131,7 @@ defmodule Cream.Connection do {:error, %Cream.Error{reason: :econnrefused, server: "localhost:99899"}} """ - @spec start_link(Keyword.t) :: {:ok, t} | {:error, reason} + @spec start_link(config) :: {:ok, t} | {:error, reason} def start_link(config \\ []) do config = Keyword.merge(config(), config) Connection.start_link(__MODULE__, config) diff --git a/lib/cream/errors/connection_error.ex b/lib/cream/errors/connection_error.ex index 901423f..aa133ee 100644 --- a/lib/cream/errors/connection_error.ex +++ b/lib/cream/errors/connection_error.ex @@ -2,10 +2,10 @@ defmodule Cream.ConnectionError do @moduledoc """ Server connection errors. - `reason` will be things like `:nxdomain` and `:econnrefused` as returned by + `:reason` will be things like `:nxdomain` and `:econnrefused` as returned by Erlang's `:gen_tcp` module. - `server` will be a server binary like `"localhost:11211"`. + `:server` will be a binary like `"localhost:11211"`. """ defexception [:reason, :server] diff --git a/lib/cream/errors/error.ex b/lib/cream/errors/error.ex index 44edebd..2bcf760 100644 --- a/lib/cream/errors/error.ex +++ b/lib/cream/errors/error.ex @@ -2,13 +2,13 @@ defmodule Cream.Error do @moduledoc """ Semantic errors returned by memcached server. - The `reason` field corresponds to a packet's status as described by + The `:reason` field corresponds to a packet's status as described by the memcached binary protocol. Notable reasons are... - * `not_found` - Key does not exist. - * `exists` - Usually a cas error. + * `:not_found` - Key does not exist. + * `:exists` - Usually a cas error. """ diff --git a/lib/cream/logger.ex b/lib/cream/logger.ex index 28d2121..bbd16d6 100644 --- a/lib/cream/logger.ex +++ b/lib/cream/logger.ex @@ -1,4 +1,5 @@ defmodule Cream.Logger do + @moduledoc false require Logger def init do From 0fab814d193e67cbde6ffb92c44b77e4aeee36e9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 25 Mar 2023 15:42:46 -0500 Subject: [PATCH 26/28] Fix tests --- Dockerfile | 5 ----- config/runtime.exs | 13 ++++++++++--- docker-compose.yml | 6 ------ test/client_test.exs | 4 +++- test/support/populate.rb | 8 ++------ 5 files changed, 15 insertions(+), 21 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8327465..0000000 --- a/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ruby:2.7-alpine - -ADD Gemfile test/support/populate.rb ./ -RUN bundle install -ENTRYPOINT [ "/bin/sleep" ] \ No newline at end of file diff --git a/config/runtime.exs b/config/runtime.exs index d424a6e..94d0407 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -2,9 +2,16 @@ import Config {json, 0} = System.cmd("docker", ~w(compose ps --format json)) -servers = Jason.decode!(json) -|> Enum.sort_by(& &1["Name"]) -|> Enum.map(fn container -> +containers = Jason.decode!(json) + +servers =[ + "cream_ex-memcached-1", + "cream_ex-memcached-2", + "cream_ex-memcached-3" +] +|> Enum.map(fn name -> + container = Enum.find(containers, & &1["Name"] == name) + port = container["Publishers"] |> List.first() |> Map.get("PublishedPort") diff --git a/docker-compose.yml b/docker-compose.yml index 8ef74fc..b23b6fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,3 @@ services: deploy: mode: replicated replicas: 3 - - dalli: - tty: true - stdin_open: true - build: - context: . \ No newline at end of file diff --git a/test/client_test.exs b/test/client_test.exs index f2f6df3..a8f181c 100644 --- a/test/client_test.exs +++ b/test/client_test.exs @@ -5,7 +5,9 @@ defmodule ClientTest do setup_all do {:ok, _client} = TestClient.start_link() :ok = TestClient.flush() - {_, 0} = System.cmd("bundle", ["exec", "ruby", "test/support/populate.rb"]) + servers = TestClient.config[:servers] + {_, 0} = System.cmd("bundle", ["exec", "ruby", "test/support/populate.rb" | servers]) + :ok end diff --git a/test/support/populate.rb b/test/support/populate.rb index 108a9cc..ad239f0 100644 --- a/test/support/populate.rb +++ b/test/support/populate.rb @@ -1,13 +1,9 @@ require "dalli" require "json" -containers = JSON.parse(`docker compose ps --format json`) -servers = containers.sort_by{ |c| c["Name"] }.map do |c| - port = c["Publishers"][0]["PublishedPort"] - "localhost:#{port}" -end +puts ARGV.inspect -client = Dalli::Client.new(servers, serializer: JSON) +client = Dalli::Client.new(ARGV.to_a, serializer: JSON) 100.times do |i| client.set("cream_ruby_test_key_#{i}", i) From 3a060d2f4108a36f9c660c5f6dedd314eeaa8c92 Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 25 Mar 2023 15:44:40 -0500 Subject: [PATCH 27/28] Bump dalli --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c45e055..25ffe3c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: https://rubygems.org/ specs: - dalli (3.2.2) + dalli (3.2.4) PLATFORMS ruby From f9de278852432ae914f7bf8f30d0240f8777e61f Mon Sep 17 00:00:00 2001 From: "Christopher J. Bottaro" Date: Sat, 22 Apr 2023 21:10:20 -0500 Subject: [PATCH 28/28] Enhancements for Superbolide --- lib/cream/coder/erlang_term.ex | 23 +++++++++++++++++++++++ lib/cream/coder/gzip.ex | 2 +- lib/cream/coder/jason.ex | 2 +- lib/cream/connection.ex | 25 +++++++++++++++++-------- lib/cream/item.ex | 2 +- mix.exs | 4 ++-- 6 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 lib/cream/coder/erlang_term.ex diff --git a/lib/cream/coder/erlang_term.ex b/lib/cream/coder/erlang_term.ex new file mode 100644 index 0000000..4a0c488 --- /dev/null +++ b/lib/cream/coder/erlang_term.ex @@ -0,0 +1,23 @@ +defmodule Cream.Coder.ErlangTerm do + @moduledoc """ + A coder using :`erlang.term_to_binary/1`. + + Uses the first bit on flags. + """ + import Bitwise + + @behaviour Cream.Coder + + def encode(value, flags) do + {:ok, :erlang.term_to_binary(value), flags ||| 0b01} + end + + def decode(value, flags) when (flags &&& 0b01) == 0b01 do + {:ok, :erlang.binary_to_term(value)} + end + + def decode(value, _flags) do + {:ok, value} + end + +end diff --git a/lib/cream/coder/gzip.ex b/lib/cream/coder/gzip.ex index ec99664..ae7fb0e 100644 --- a/lib/cream/coder/gzip.ex +++ b/lib/cream/coder/gzip.ex @@ -4,7 +4,7 @@ defmodule Cream.Coder.Gzip do It's uses the second bit on flags. """ - use Bitwise + import Bitwise @behaviour Cream.Coder diff --git a/lib/cream/coder/jason.ex b/lib/cream/coder/jason.ex index 13e111f..a5865ce 100644 --- a/lib/cream/coder/jason.ex +++ b/lib/cream/coder/jason.ex @@ -4,7 +4,7 @@ defmodule Cream.Coder.Jason do Uses the first bit on flags. """ - use Bitwise + import Bitwise @behaviour Cream.Coder diff --git a/lib/cream/connection.ex b/lib/cream/connection.ex index 3452d3d..8e065be 100644 --- a/lib/cream/connection.ex +++ b/lib/cream/connection.ex @@ -350,21 +350,21 @@ defmodule Cream.Connection do {:ok, "bar"} """ - @spec fetch(t, binary, Keyword.t, (-> term)) :: {:ok, term} | {:error, Error.t} + @spec fetch(t, binary, Keyword.t, (-> term)) :: term def fetch(conn, key, opts \\ [], f) do case get(conn, key, Keyword.put(opts, :quiet, false)) do - {:ok, value} -> {:ok, value} + {:ok, value} -> value {:error, %Error{reason: :not_found}} -> value = f.() case set(conn, {key, value}, opts) do - :ok -> {:ok, value} - {:ok, _item} -> {:ok, value} - {:error, %ConnectionError{}} -> {:ok, value} + :ok -> value + {:ok, _item} -> value + {:error, %ConnectionError{}} -> value error -> error end - {:error, %ConnectionError{}} -> {:ok, f.()} - error -> error + {:error, %ConnectionError{}} -> f.() + _error -> f.() end end @@ -485,7 +485,16 @@ defmodule Cream.Connection do end def handle_call({:set, item}, from, state) do - %{socket: socket} = state + %{config: config, socket: socket} = state + + item = if item.ttl == nil do + case config[:ttl] do + nil -> %{item | ttl: 0} + ttl -> %{item | ttl: ttl} + end + else + item + end packet = Protocol.set(item) diff --git a/lib/cream/item.ex b/lib/cream/item.ex index 33b1d74..199acea 100644 --- a/lib/cream/item.ex +++ b/lib/cream/item.ex @@ -14,7 +14,7 @@ defmodule Cream.Item do been applied. It can be any kind of term. """ - defstruct [:key, :value, :raw_value, ttl: 0, cas: 0, flags: 0, coder: nil] + defstruct [:key, :value, :raw_value, ttl: nil, cas: 0, flags: 0, coder: nil] @type t :: %__MODULE__{ key: binary, diff --git a/mix.exs b/mix.exs index d0eda5f..92c1d08 100644 --- a/mix.exs +++ b/mix.exs @@ -60,9 +60,9 @@ defmodule Cream.Mixfile do # Type "mix help deps" for more examples and options defp deps do [ - {:telemetry, "~> 0.4.2"}, + {:telemetry, "~> 0.4.2 or ~> 1.0"}, {:connection, "~> 1.0"}, - {:nimble_pool, "~> 0.0"}, + {:nimble_pool, "~> 0.2 or ~> 1.0"}, {:jason, "~> 1.0", only: [:dev, :test]}, {:ex_doc, "~> 0.0", only: :dev}, ]