Replaced refs with snowflakes (#536)

Despite my idea of creating refs to represent IDs of syntactic
elements, they are not globally unique, and can be reused when the VM
restarts. This makes them inappropriate for long-lived ids, like we
need in the index.

I've replaced them with snowflakes, which i benchmarked at 800k qps,
are unique, and also represent the time they were created, which means
we'll not need to store dates in the index. After the change,
lexical's index was 50 MB smaller.

Author: scohen

apps/common/lib/lexical/ast/analysis.ex

@@ -12,6 +12,7 @@ defmodule Lexical.Ast.Analysis do
   alias Lexical.Document
   alias Lexical.Document
   alias Lexical.Document.Position
+  alias Lexical.Identifier
   alias Sourceror.Zipper
 
   defstruct [:ast, :document, :parse_error, scopes: [], valid?: true]
@@ -146,7 +147,8 @@ defmodule Lexical.Ast.Analysis do
 
   # add a unique ID to 3-element tuples
   defp with_scope_id({_, _, _} = quoted) do
-    Macro.update_meta(quoted, &Keyword.put(&1, @scope_id, make_ref()))
+    id = Identifier.next_global!()
+    Macro.update_meta(quoted, &Keyword.put(&1, @scope_id, id))
   end
 
   defp with_scope_id(quoted) do

apps/common/lib/lexical/identifier.ex

@@ -0,0 +1,10 @@
+defmodule Lexical.Identifier do
+  @doc """
+  Returns the next globally unique identifier.
+  Raises a MatchError if this cannot be computed.
+  """
+  def next_global! do
+    {:ok, next_id} = Snowflake.next_id()
+    next_id
+  end
+end

apps/common/mix.exs

@@ -34,6 +34,7 @@ defmodule Common.MixProject do
     [
       {:lexical_shared, path: "../../projects/lexical_shared"},
       {:lexical_test, path: "../../projects/lexical_test", only: :test},
+      {:snowflake, "~> 1.0"},
       {:sourceror, "~> 0.14.1"},
       {:stream_data, "~> 0.6", only: [:test], runtime: false},
       {:patch, "~> 0.12", only: [:test], optional: true, runtime: false}

apps/common/test/test_helper.exs

@@ -1 +1,2 @@
+Application.ensure_all_started(:snowflake)
 ExUnit.start()

apps/remote_control/benchmarks/data/source.index.v1.ets

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8b938d9c25331df2752ecf44b1b7823f80e68a0c34489aa0df58edd75d096ee
-size 99882388
+oid sha256:6aa5313736d6431963476b7b0b38cc20f2c95b252b3358ca6fba17b8d116d6a0
+size 221782947

apps/remote_control/benchmarks/ets_bench.exs

@@ -3,6 +3,7 @@ alias Lexical.RemoteControl
 alias Lexical.RemoteControl.Search.Store.Backends.Ets
 alias Lexical.RemoteControl.Search.Store.Backends.Ets.Schema
 alias Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas
+alias Lexical.VM.Versions
 
 defmodule BenchHelper do
   def wait_for_registration do
@@ -20,14 +21,14 @@ defmodule BenchHelper do
     Enum.random(entries).path
   end
 
-  def random_ref(entries) do
-    Enum.random(entries).ref
+  def random_id(entries) do
+    Enum.random(entries).id
   end
 
-  def random_refs(entries, count) do
+  def random_ids(entries, count) do
     entries
     |> Enum.take_random(count)
-    |> Enum.map(& &1.ref)
+    |> Enum.map(& &1.id)
   end
 end
 
@@ -37,7 +38,8 @@ project = Project.new("file://#{cwd}")
 RemoteControl.set_project(project)
 Project.ensure_workspace(project)
 
-indexes_path = Project.workspace_path(project, "indexes")
+versions = Versions.current()
+indexes_path = Project.workspace_path(project, ["indexes", versions.erlang, versions.elixir])
 data_dir = Path.join(cwd, "data")
 
 File.mkdir_p!(indexes_path)
@@ -64,17 +66,17 @@ Benchee.run(
     "find_by_subject, two wildcards" => fn _ ->
       Ets.find_by_subject(Enum, :_, :_)
     end,
-    "find_by_references" => fn %{refs: refs} ->
-      Ets.find_by_refs(refs, :module, :_)
+    "find_by_references" => fn %{ids: ids} ->
+      Ets.find_by_ids(ids, :module, :_)
     end,
     "delete_by_path" => fn %{path: path} ->
       Ets.delete_by_path(path)
     end
   },
   before_each: fn _ ->
-    refs = BenchHelper.random_refs(entries, 50)
+    ids = BenchHelper.random_ids(entries, 50)
     path = BenchHelper.random_path(entries)
-    %{path: path, refs: refs}
+    %{path: path, ids: ids}
   end
 )
 

apps/remote_control/benchmarks/snowflake_bench.exs

@@ -0,0 +1,8 @@
+Benchee.run(
+  %{
+    "next_id" => fn ->
+      Snowflake.next_id()
+    end
+  },
+  profile_after: true
+)

apps/remote_control/lib/lexical/remote_control/search/fuzzy.ex

@@ -39,7 +39,7 @@ defmodule Lexical.RemoteControl.Search.Fuzzy do
   @spec from_entries([Entry.t()]) :: t
   def from_entries(entries) do
     mapper = fn %Entry{} = entry ->
-      {entry.subject, entry.path, entry.ref}
+      {entry.subject, entry.path, entry.id}
     end
 
     new(entries, mapper, &stringify/1)
@@ -85,13 +85,13 @@ defmodule Lexical.RemoteControl.Search.Fuzzy do
   in descending order of the match relevance. Items at the beginning of the list
   will have a higher score than items at the end.
   """
-  @spec match(t(), String.t()) :: [reference()]
+  @spec match(t(), String.t()) :: [Entry.entry_id()]
   def match(%__MODULE__{} = fuzzy, pattern) do
     fuzzy.subject_to_values
-    |> Stream.map(fn {subject, references} ->
+    |> Stream.map(fn {subject, ids} ->
       case score(fuzzy, subject, pattern) do
         {:ok, score} ->
-          {score, references}
+          {score, ids}
 
         :error ->
           nil
@@ -117,13 +117,13 @@ defmodule Lexical.RemoteControl.Search.Fuzzy do
     subject = fuzzy.subject_converter.(extracted_subject)
 
     updated_grouping_key_to_values =
-      Map.update(fuzzy.grouping_key_to_values, grouping_key, [value], fn old_refs ->
-        [value | old_refs]
+      Map.update(fuzzy.grouping_key_to_values, grouping_key, [value], fn old_ids ->
+        [value | old_ids]
       end)
 
     updated_subject_to_values =
-      Map.update(fuzzy.subject_to_values, subject, [value], fn old_refs ->
-        [value | old_refs]
+      Map.update(fuzzy.subject_to_values, subject, [value], fn old_ids ->
+        [value | old_ids]
       end)
 
     updated_preprocessed_subjects =

apps/remote_control/lib/lexical/remote_control/search/indexer/entry.ex

@@ -3,14 +3,14 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
   @type subject :: String.t()
   @type entry_subtype :: :reference | :definition
   @type version :: String.t()
-  @type entry_reference :: reference() | nil
+  @type entry_id :: pos_integer() | nil
 
   defstruct [
     :application,
+    :id,
     :parent,
     :path,
     :range,
-    :ref,
     :subject,
     :subtype,
     :type,
@@ -20,10 +20,9 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
   @type t :: %__MODULE__{
           application: module(),
           subject: subject(),
-          parent: entry_reference(),
+          parent: entry_id(),
           path: Path.t(),
           range: Lexical.Document.Range.t(),
-          ref: entry_reference(),
           subtype: entry_subtype(),
           type: entry_type(),
           updated_at: :calendar.datetime()
@@ -33,22 +32,22 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
 
   use StructAccess
 
-  def reference(path, ref, parent, subject, type, range, application) do
-    new(path, ref, parent, subject, type, :reference, range, application)
+  def reference(path, id, parent, subject, type, range, application) do
+    new(path, id, parent, subject, type, :reference, range, application)
   end
 
-  def definition(path, ref, parent, subject, type, range, application) do
-    new(path, ref, parent, subject, type, :definition, range, application)
+  def definition(path, id, parent, subject, type, range, application) do
+    new(path, id, parent, subject, type, :definition, range, application)
   end
 
-  defp new(path, ref, parent, subject, type, subtype, range, application) do
+  defp new(path, id, parent, subject, type, subtype, range, application) do
     %__MODULE__{
       application: application,
       subject: subject,
+      id: id,
       parent: parent,
       path: path,
       range: range,
-      ref: ref,
       subtype: subtype,
       type: type,
       updated_at: timestamp()

apps/remote_control/lib/lexical/remote_control/search/indexer/extractors/function_definition.ex

@@ -3,6 +3,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.FunctionDefinition do
   alias Lexical.Document.Position
   alias Lexical.Document.Range
   alias Lexical.Formats
+  alias Lexical.Identifier
   alias Lexical.RemoteControl
   alias Lexical.RemoteControl.Search.Indexer.Entry
   alias Lexical.RemoteControl.Search.Indexer.Metadata
@@ -39,8 +40,8 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.FunctionDefinition do
     entry =
       Entry.definition(
         reducer.analysis.document.path,
-        make_ref(),
-        block.parent_ref,
+        Identifier.next_global!(),
+        block.parent_id,
         mfa,
         type,
         range,

apps/remote_control/lib/lexical/remote_control/search/indexer/extractors/function_reference.ex

@@ -2,6 +2,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.FunctionReference do
   alias Lexical.Document.Position
   alias Lexical.Document.Range
   alias Lexical.Formats
+  alias Lexical.Identifier
   alias Lexical.RemoteControl
   alias Lexical.RemoteControl.Search.Indexer.Entry
   alias Lexical.RemoteControl.Search.Indexer.Metadata
@@ -140,8 +141,8 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.FunctionReference do
 
         Entry.reference(
           reducer.analysis.document.path,
-          make_ref(),
-          block.parent_ref,
+          Identifier.next_global!(),
+          block.parent_id,
           mfa,
           :function,
           range,

apps/remote_control/lib/lexical/remote_control/search/indexer/extractors/module.ex

@@ -6,6 +6,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.Module do
   alias Lexical.Ast
   alias Lexical.Document.Position
   alias Lexical.Document.Range
+  alias Lexical.Identifier
   alias Lexical.ProcessCache
   alias Lexical.RemoteControl
   alias Lexical.RemoteControl.Search.Indexer.Entry
@@ -31,8 +32,8 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.Module do
         entry =
           Entry.definition(
             reducer.analysis.document.path,
-            block.ref,
-            block.parent_ref,
+            block.id,
+            block.parent_id,
             aliased_module,
             :module,
             range,
@@ -64,8 +65,8 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.Module do
         entry =
           Entry.reference(
             reducer.analysis.document.path,
-            make_ref(),
-            current_block.ref,
+            Identifier.next_global!(),
+            current_block.id,
             module,
             :module,
             range,
@@ -91,8 +92,8 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.Module do
         entry =
           Entry.reference(
             reducer.analysis.document.path,
-            make_ref(),
-            current_block.ref,
+            Identifier.next_global!(),
+            current_block.id,
             module,
             :module,
             range,

apps/remote_control/lib/lexical/remote_control/search/indexer/source/block.ex

@@ -2,13 +2,16 @@ defmodule Lexical.RemoteControl.Search.Indexer.Source.Block do
   @moduledoc """
   A struct that represents a block of source code
   """
-  defstruct [:starts_at, :ends_at, :ref, :parent_ref]
+
+  defstruct [:starts_at, :ends_at, :id, :parent_id]
+  alias Lexical.Identifier
 
   def root do
-    %__MODULE__{ref: :root}
+    %__MODULE__{id: :root}
   end
 
   def new(starts_at, ends_at) do
-    %__MODULE__{starts_at: starts_at, ends_at: ends_at, ref: make_ref()}
+    id = Identifier.next_global!()
+    %__MODULE__{starts_at: starts_at, ends_at: ends_at, id: id}
   end
 end

apps/remote_control/lib/lexical/remote_control/search/indexer/source/reducer.ex

@@ -109,20 +109,20 @@ defmodule Lexical.RemoteControl.Search.Indexer.Source.Reducer do
 
   defp push_block(%__MODULE__{} = reducer, %Block{} = block) do
     parent = List.first(reducer.blocks)
-    block = %Block{block | parent_ref: parent.ref}
+    block = %Block{block | parent_id: parent.id}
     %__MODULE__{reducer | blocks: [block | reducer.blocks]}
   end
 
   # you never pop the root block in a document
-  defp pop_block(%__MODULE__{blocks: [%Block{ref: :root}]} = reducer), do: reducer
+  defp pop_block(%__MODULE__{blocks: [%Block{id: :root}]} = reducer), do: reducer
 
   defp pop_block(%__MODULE__{} = reducer) do
     [_ | rest] = reducer.blocks
     %__MODULE__{reducer | blocks: rest}
   end
 
   # The root block in the document goes on forever
-  defp block_ended?(%__MODULE__{blocks: [%Block{ref: :root}]}), do: false
+  defp block_ended?(%__MODULE__{blocks: [%Block{id: :root}]}), do: false
 
   defp block_ended?(%__MODULE__{} = reducer) do
     %Block{} = block = current_block(reducer)

apps/remote_control/lib/lexical/remote_control/search/store/backend.ex

@@ -67,7 +67,7 @@ defmodule Lexical.RemoteControl.Search.Store.Backend do
   @doc """
   Deletes all entries whose path is equal to the one passed in.
   """
-  @callback delete_by_path(Path.t()) :: {:ok, [reference()]} | {:error, any()}
+  @callback delete_by_path(Path.t()) :: {:ok, [Entry.entry_id()]} | {:error, any()}
 
   @doc """
   Finds all entries
@@ -77,7 +77,7 @@ defmodule Lexical.RemoteControl.Search.Store.Backend do
   @doc """
   Finds entries whose ref attribute is in the given list
   """
-  @callback find_by_refs([reference()], type_query(), subtype_query()) :: [Entry.t()]
+  @callback find_by_ids([Entry.entry_id()], type_query(), subtype_query()) :: [Entry.t()]
 
   @optional_callbacks sync: 1
 end

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets.ex

@@ -69,8 +69,8 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets do
   end
 
   @impl Backend
-  def find_by_refs(references, type, subtype) do
-    GenServer.call(genserver_name(), {:find_by_references, [references, type, subtype]})
+  def find_by_ids(ids, type, subtype) do
+    GenServer.call(genserver_name(), {:find_by_ids, [ids, type, subtype]})
   end
 
   def start_link(%Project{} = project) do