Improved lexical's index efficiency (#534)

For lexical, the index was using 450MB of disk and consumed about a
gig in memory. This commit includes the following changes:

  * We stopped duplicating entries in a couple of the records, and now
    only store them once in the 'by_ref' key. This decreased the index
    size by 50%, and had no noticable effect on performance.
  * Several of the processes had cache entries lying around that
    caused them to consume quite a bit (hundreds of megs) of memory. I
    introduced a cache cleaning block and added it to the call sites,
    and this reduced the memory usage of those processes back to
    extremely low levels.
  * We were storing the erlang and elixir versions on each entry. Now we
    create separate indexes for each elixir/erlang version pair

In total, lexical was taking around 1gb of memory before these
changes, and now it takes around 400mb. Disk space dropped from 450 megs
to 250.

Author: scohen

apps/remote_control/lib/lexical/remote_control/commands/reindex.ex

@@ -2,10 +2,13 @@ defmodule Lexical.RemoteControl.Commands.Reindex do
   defmodule State do
     alias Lexical.Ast.Analysis
     alias Lexical.Document
+    alias Lexical.ProcessCache
     alias Lexical.RemoteControl.Search
     alias Lexical.RemoteControl.Search.Indexer
 
     require Logger
+    require ProcessCache
+
     defstruct reindex_fun: nil, index_task: nil, pending_updates: %{}
 
     def new(reindex_fun) do
@@ -53,7 +56,7 @@ defmodule Lexical.RemoteControl.Commands.Reindex do
     defp entries_for_uri(uri) do
       with {:ok, %Document{} = document, %Analysis{} = analysis} <-
              Document.Store.fetch(uri, :analysis),
-           {:ok, entries} <- Indexer.Quoted.index(analysis) do
+           {:ok, entries} <- Indexer.Quoted.index_with_cleanup(analysis) do
         {:ok, document.path, entries}
       else
         error ->

apps/remote_control/lib/lexical/remote_control/search/indexer.ex

@@ -1,21 +1,32 @@
 defmodule Lexical.RemoteControl.Search.Indexer do
+  alias Lexical.ProcessCache
   alias Lexical.Project
   alias Lexical.RemoteControl.Search.Indexer
 
   import Lexical.RemoteControl.Progress
+  require ProcessCache
+
   @indexable_extensions "*.{ex,exs}"
 
   def create_index(%Project{} = project) do
-    entries =
-      project
-      |> indexable_files()
-      |> async_chunks(&index_path/1)
-      |> List.flatten()
-
-    {:ok, entries}
+    ProcessCache.with_cleanup do
+      entries =
+        project
+        |> indexable_files()
+        |> async_chunks(&index_path/1)
+        |> List.flatten()
+
+      {:ok, entries}
+    end
   end
 
   def update_index(%Project{} = project, existing_entries) do
+    ProcessCache.with_cleanup do
+      do_update_index(project, existing_entries)
+    end
+  end
+
+  defp do_update_index(%Project{} = project, existing_entries) do
     path_to_last_index_at =
       existing_entries
       |> Enum.group_by(& &1.path, & &1.updated_at)

apps/remote_control/lib/lexical/remote_control/search/indexer/entry.ex

@@ -7,8 +7,6 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
 
   defstruct [
     :application,
-    :elixir_version,
-    :erlang_version,
     :parent,
     :path,
     :range,
@@ -21,8 +19,6 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
 
   @type t :: %__MODULE__{
           application: module(),
-          elixir_version: version(),
-          erlang_version: version(),
           subject: subject(),
           parent: entry_reference(),
           path: Path.t(),
@@ -34,7 +30,6 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
         }
 
   alias Lexical.StructAccess
-  alias Lexical.VM.Versions
 
   use StructAccess
 
@@ -47,12 +42,8 @@ defmodule Lexical.RemoteControl.Search.Indexer.Entry do
   end
 
   defp new(path, ref, parent, subject, type, subtype, range, application) do
-    versions = Versions.current()
-
     %__MODULE__{
       application: application,
-      elixir_version: versions.elixir,
-      erlang_version: versions.erlang,
       subject: subject,
       parent: parent,
       path: path,

apps/remote_control/lib/lexical/remote_control/search/indexer/extractors/function_definition.ex

@@ -39,7 +39,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.FunctionDefinition do
     entry =
       Entry.definition(
         reducer.analysis.document.path,
-        block.ref,
+        make_ref(),
         block.parent_ref,
         mfa,
         type,

apps/remote_control/lib/lexical/remote_control/search/indexer/extractors/function_reference.ex

@@ -140,7 +140,7 @@ defmodule Lexical.RemoteControl.Search.Indexer.Extractors.FunctionReference do
 
         Entry.reference(
           reducer.analysis.document.path,
-          block.ref,
+          make_ref(),
           block.parent_ref,
           mfa,
           :function,

apps/remote_control/lib/lexical/remote_control/search/indexer/quoted.ex

@@ -1,7 +1,16 @@
 defmodule Lexical.RemoteControl.Search.Indexer.Quoted do
   alias Lexical.Ast.Analysis
+  alias Lexical.ProcessCache
   alias Lexical.RemoteControl.Search.Indexer.Source.Reducer
 
+  require ProcessCache
+
+  def index_with_cleanup(%Analysis{} = analysis) do
+    ProcessCache.with_cleanup do
+      index(analysis)
+    end
+  end
+
   def index(%Analysis{valid?: true} = analysis) do
     {_, reducer} =
       Macro.prewalk(analysis.ast, Reducer.new(analysis), fn elem, reducer ->

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets/schema.ex

@@ -55,6 +55,7 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schema do
   end
 
   alias Lexical.Project
+  alias Lexical.VM.Versions
 
   def load(%Project{} = project, schema_order) do
     ensure_unique_versions(schema_order)
@@ -89,7 +90,9 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schema do
   end
 
   def index_root(%Project{} = project) do
-    Project.workspace_path(project, "indexes")
+    versions = Versions.current()
+    index_path = ["indexes", versions.erlang, versions.elixir]
+    Project.workspace_path(project, index_path)
   end
 
   def index_file_path(%Project{} = project, schema) do

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets/schemas/v1.ex

@@ -14,14 +14,12 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas.V1 do
 
   use Schema, version: 1
 
-  defkey :by_id, [:id, :type, :subtype, :elixir_version, :erlang_version]
+  defkey :by_id, [:id, :type, :subtype]
 
   defkey :by_subject, [
     :subject,
     :type,
     :subtype,
-    :elixir_version,
-    :erlang_version,
     :path
   ]
 
@@ -31,7 +29,7 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas.V1 do
     migrated =
       entries
       |> Stream.filter(fn
-        {_, %_{elixir_version: _, erlang_version: _, type: _, subtype: _, ref: _}} -> true
+        {_, %_{type: _, subtype: _, ref: _}} -> true
         _ -> false
       end)
       |> Stream.map(fn {_, entry} -> entry end)
@@ -53,8 +51,6 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas.V1 do
   def to_rows(%Entry{} = entry) do
     subject_key =
       by_subject(
-        elixir_version: entry.elixir_version,
-        erlang_version: entry.erlang_version,
         subject: to_subject(entry.subject),
         type: entry.type,
         subtype: entry.subtype,
@@ -65,18 +61,16 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas.V1 do
       by_id(
         id: entry.ref,
         type: entry.type,
-        subtype: entry.subtype,
-        elixir_version: entry.elixir_version,
-        erlang_version: entry.erlang_version
+        subtype: entry.subtype
       )
 
     path_key = by_path(path: entry.path)
 
-    [{subject_key, entry}, {id_key, entry}, {path_key, id_key}]
+    [{subject_key, id_key}, {id_key, entry}, {path_key, id_key}]
   end
 
   # This case will handle any namespaced entries
-  def to_rows(%{elixir_version: _, erlang_version: _, type: _, subtype: _, ref: _} = entry) do
+  def to_rows(%{type: _, subtype: _, ref: _} = entry) do
     map = Map.delete(entry, :__struct__)
 
     Entry

apps/remote_control/lib/lexical/remote_control/search/store/backends/ets/state.ex

@@ -8,7 +8,6 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.State do
   alias Lexical.Project
   alias Lexical.RemoteControl.Search.Store.Backends.Ets.Schema
   alias Lexical.RemoteControl.Search.Store.Backends.Ets.Schemas
-  alias Lexical.VM.Versions
 
   @schema_order [
     Schemas.LegacyV0,
@@ -60,28 +59,26 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.State do
   end
 
   def find_by_subject(%__MODULE__{} = state, subject, type, subtype) do
-    versions = Versions.current()
-
     match_pattern =
       query_by_subject(
         subject: to_subject(subject),
         type: type,
-        subtype: subtype,
-        elixir_version: versions.elixir,
-        erlang_version: versions.erlang
+        subtype: subtype
       )
 
     state.table_name
     |> :ets.match_object({match_pattern, :_})
-    |> Enum.flat_map(fn {_, match} -> match end)
+    |> Enum.flat_map(fn {_, id_keys} ->
+      id_keys
+    end)
+    |> MapSet.new()
+    |> Enum.flat_map(&:ets.lookup_element(state.table_name, &1, 2))
   end
 
   def find_by_references(%__MODULE__{} = state, references, type, subtype)
       when is_list(references) do
-    versions = Versions.current()
-
     for reference <- references,
-        match_pattern = match_id_key(reference, versions, type, subtype),
+        match_pattern = match_id_key(reference, type, subtype),
         {_key, entry} <- :ets.match_object(state.table_name, match_pattern) do
       entry
     end
@@ -133,13 +130,11 @@ defmodule Lexical.RemoteControl.Search.Store.Backends.Ets.State do
     state
   end
 
-  defp match_id_key(reference, versions, type, subtype) do
+  defp match_id_key(reference, type, subtype) do
     {query_by_id(
        id: reference,
        type: type,
-       subtype: subtype,
-       elixir_version: versions.elixir,
-       erlang_version: versions.erlang
+       subtype: subtype
      ), :_}
   end
 

apps/remote_control/test/lexical/remote_control/search/store_test.exs

@@ -62,29 +62,6 @@ defmodule Lexical.RemoteControl.Search.StoreTest do
         assert ref.subtype == :reference
       end
 
-      test "matching can exclude on elixir version" do
-        Store.replace([
-          reference(subject: Enum, elixir_version: "1.0.0"),
-          reference(subject: Enum)
-        ])
-
-        assert {:ok, [ref]} = Store.exact("Enum", subtype: :reference)
-        assert ref.subject == Enum
-        refute ref.elixir_version == "1.0.0"
-      end
-
-      test "matching can exclude on erlang version" do
-        Store.replace([
-          reference(subject: Enum, erlang_version: "1.0.0"),
-          reference(subject: Enum)
-        ])
-
-        assert {:ok, [ref]} = Store.exact("Enum", subtype: :reference)
-
-        assert ref.subject == Enum
-        refute ref.erlang_version == "1.0.0"
-      end
-
       test "matching with queries can exclude on type" do
         Store.replace([
           reference(subject: Foo.Bar.Baz),
@@ -124,26 +101,6 @@ defmodule Lexical.RemoteControl.Search.StoreTest do
         assert entry_1.subject in [Foo.Bar.Baz, Foo.Bar.Bak]
         assert entry_2.subject in [Foo.Bar.Baz, Foo.Bar.Bak]
       end
-
-      test "matching only returns entries specific to our elixir version" do
-        Store.replace([
-          definition(ref: 1, subject: Foo.Bar.Baz, elixir_version: "1.1"),
-          definition(ref: 2, subject: Foo.Bar.Baz)
-        ])
-
-        assert {:ok, [entry]} = Store.fuzzy("Foo.Bar.", type: :module, subtype: :definition)
-        assert entry.ref == 2
-      end
-
-      test "matching only returns entries specific to our erlang version" do
-        Store.replace([
-          definition(ref: 1, subject: Foo.Bar.Baz, erlang_version: "14.3.2.8"),
-          definition(ref: 2, subject: Foo.Bar.Baz)
-        ])
-
-        assert {:ok, [entry]} = Store.fuzzy("Foo.Bar.", type: :module, subtype: :definition)
-        assert entry.ref == 2
-      end
     end
 
     describe "#{backend_name} :: updating entries in a file" do

apps/remote_control/test/support/lexical/test/entry/entry_builder.ex

@@ -1,20 +1,15 @@
 defmodule Lexical.Test.Entry.Builder do
   alias Lexical.Document.Range
   alias Lexical.RemoteControl.Search.Indexer.Entry
-  alias Lexical.VM.Versions
 
   import Lexical.Test.PositionSupport
 
   def entry(fields \\ []) do
-    versions = Versions.current()
-
     defaults = [
       subject: Module,
       ref: make_ref(),
       path: "/foo/bar/baz.ex",
       range: range(1, 1, 1, 5),
-      elixir_version: versions.elixir,
-      erlang_version: versions.erlang,
       type: :module
     ]
 

projects/lexical_shared/lib/lexical/process_cache.ex

@@ -58,6 +58,12 @@ defmodule Lexical.ProcessCache do
     end
   end
 
+  def clear_keys do
+    keys()
+    |> MapSet.put(all_keys_key())
+    |> Enum.each(&Process.delete/1)
+  end
+
   @doc """
   Retrieves and optionally sets a value in the cache.
 
@@ -76,9 +82,35 @@ defmodule Lexical.ProcessCache do
     end
   end
 
+  defmacro with_cleanup(do: block) do
+    quote do
+      try do
+        unquote(block)
+      after
+        unquote(__MODULE__).clear_keys()
+      end
+    end
+  end
+
   defp set(key, timeout_ms, compute_fn) do
     value = compute_fn.()
+
+    add_key(key)
     Process.put(key, Entry.new(value, timeout_ms))
+
     value
   end
+
+  defp add_key(key) do
+    updated_keys = MapSet.put(keys(), key)
+    Process.put(all_keys_key(), updated_keys)
+  end
+
+  defp all_keys_key do
+    {__MODULE__, :all_keys}
+  end
+
+  defp keys do
+    Process.get(all_keys_key(), MapSet.new())
+  end
 end