New project structure / beginning of plugins (#184)

* New project structure / beginning of plugins

Making plugins necessitated a new project structure, as we're going to
need to publish a separate library for plugin authors so they can use
them. We'll also need a bunch of the modules they'll use, so we need
the least stressful way of depending on the libraries we publish.

Creating a separate repo would make our lives very difficult, so I
added a projects directory at the top level, and the published
libraries go in there. We utilize path dependencies for our internal
libraries.

I also added the beginnings of a plugin architecture, but I wanted to
get this change out before it gets too big. Plugins are quite
complicated, and we're going to need to do a bit of refactoring when
we add one. That refactoring, plus all of this moving would make the
PR very difficult to review.

This PR also includes a new genserver dispatcher so that the compile
tracer isn't slowed down by extraneous work.

* Renamed lexical library to lexical_shared

The name lexical conflicted with the language server's app and
formatting would fail because it was using lexical's mix project. This
was also reflected by the necessity of creating a unique name for the
project, the fix was just masking the problem.

* Made best-effort to find formatter options

If we can't find the formatter, we were running the formatter with
default options, which would be annoying if you've configured
the formatter, and you'd just get the defaults. This could be
annoying. This change makes a best effort to find the .formatter.exs
in the parent directories of your application, and it stops at the app
root. If it's not found in any of the parents, you get default options.

* Automated included apps

I kept having to update the apps at the top of this file whenever I
added, removed or renamed a dependency. This is annoying, and I
thought it would be a lot easier to just let the mix.exs control which
apps are needed by remote_control.

.github/workflows/elixir.yml

@@ -85,9 +85,8 @@ jobs:
         run: mix credo
 
       - name: Compile
-        run: |
-          mix clean
-          mix compile --warnings-as-errors
+        run: make compile.all
+
 
       - name: Maybe create plt files
         if: steps.cache-plt.outputs.cache-hit != 'true'
@@ -98,8 +97,8 @@ jobs:
       - name: Run dialyzer
         run: |
           mix compile.protocols
-          mix dialyzer
+          make dialyzer.all
 
       # Step: Execute the tests.
       - name: Run tests
-        run: mix test
+        run: make test.all

Makefile

@@ -0,0 +1,29 @@
+project_dirs = lexical_shared lexical_plugin lexical_test
+dialyzer_dirs = lexical_shared lexical_plugin
+
+compile.all: compile.projects compile.umbrella
+
+dialyzer.all: compile.all dialyzer.projects dialyzer.umbrella
+
+test.all: test.projects test.umbrella
+
+dialyzer.umbrella:
+	mix dialyzer
+
+dialyzer.projects:
+	$(foreach dir, $(dialyzer_dirs), cd projects/$(dir) && mix dialyzer && cd ../..;)
+
+test.umbrella:
+	mix test
+
+test.projects:
+	cd projects
+	$(foreach dir, $(project_dirs), cd projects/$(dir) && mix test && cd ../..;)
+
+compile.umbrella: compile.projects
+	mix deps.get
+	mix compile --skip-umbrella-children
+
+compile.projects:
+	cd projects
+	$(foreach dir, $(project_dirs), cd projects/$(dir) && mix deps.get && mix do clean, compile --warnings-as-errors && cd ../..;)

apps/common/mix.exs

@@ -32,6 +32,7 @@ defmodule Common.MixProject do
 
   defp deps do
     [
+      {:lexical_shared, path: "../../projects/lexical_shared"},
       {:stream_data, "~> 0.5", only: [:test], runtime: false},
       {:patch, "~> 0.12", only: [:test], optional: true, runtime: false}
     ]

apps/proto/mix.exs

@@ -26,6 +26,7 @@ defmodule Proto.MixProject do
   defp deps do
     [
       {:jason, "~> 1.4", optional: true},
+      {:lexical_shared, path: "../../projects/lexical_shared"},
       {:common, in_umbrella: true}
     ]
   end

apps/protocol/mix.exs

@@ -30,6 +30,8 @@ defmodule Lexical.Protocol.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
+      {:lexical_shared, path: "../../projects/lexical_shared"},
+      {:lexical_test, path: "../../projects/lexical_test", only: :test},
       {:common, in_umbrella: true},
       {:jason, "~> 1.4", optional: true},
       {:patch, "~> 0.12", only: [:test]},

apps/remote_control/lib/lexical/remote_control.ex

@@ -9,7 +9,8 @@ defmodule Lexical.RemoteControl do
   alias Lexical.RemoteControl.ProjectNode
   require Logger
 
-  @allowed_apps ~w(common path_glob remote_control elixir_sense sourceror)a
+  @excluded_apps [:patch, :nimble_parsec]
+  @allowed_apps [:remote_control | Mix.Project.deps_apps()] -- @excluded_apps
 
   @app_globs Enum.map(@allowed_apps, fn app_name -> "/**/#{app_name}*/ebin" end)
 

apps/remote_control/lib/lexical/remote_control/api.ex

@@ -53,4 +53,8 @@ defmodule Lexical.RemoteControl.Api do
       position
     ])
   end
+
+  def diagnose(%Project{} = project, %Document{} = document) do
+    RemoteControl.call(project, RemoteControl.Plugin, :diagnose, [document])
+  end
 end

apps/remote_control/lib/lexical/remote_control/api/messages.ex

@@ -11,7 +11,7 @@ defmodule Lexical.RemoteControl.Api.Messages do
     diagnostics: [],
     elapsed_ms: 0
 
-  defrecord :module_updated, name: nil, functions: [], macros: [], struct: nil
+  defrecord :module_updated, file: nil, name: nil, functions: [], macros: [], struct: nil
 
   defrecord :project_diagnostics, project: nil, diagnostics: []
   defrecord :file_diagnostics, project: nil, uri: nil, diagnostics: []

apps/remote_control/lib/lexical/remote_control/application.ex

@@ -14,7 +14,9 @@ defmodule Lexical.RemoteControl.Application do
         [
           RemoteControl.ModuleMappings,
           RemoteControl.Build,
-          RemoteControl.Build.CaptureServer
+          RemoteControl.Build.CaptureServer,
+          RemoteControl.Compilation.Dispatch,
+          RemoteControl.Plugin.Supervisor
         ]
       else
         []

apps/remote_control/lib/lexical/remote_control/build/progress.ex

@@ -4,14 +4,11 @@ defmodule Lexical.RemoteControl.Build.Progress do
 
   defmacro __using__(_) do
     quote do
-      import unquote(__MODULE__), only: [with_progress: 2, with_progress: 3]
+      import unquote(__MODULE__), only: [with_progress: 2]
     end
   end
 
-  def with_progress(label, func, opts \\ []) when is_function(func, 0) do
-    mix? = Keyword.get(opts, :mix?, true)
-    label = if mix?, do: "mix " <> label, else: label
-
+  def with_progress(label, func) when is_function(func, 0) do
     try do
       RemoteControl.notify_listener(project_progress(label: label, stage: :begin))
       func.()

apps/remote_control/lib/lexical/remote_control/build/state.ex

@@ -42,23 +42,25 @@ defmodule Lexical.RemoteControl.Build.State do
     # If the project directory isn't there, for some reason the main build fails, but a
     # non-forced build will work, after which the project can be built correctly.
     project = state.project
+    build_path = Project.build_path(project)
 
-    unless File.exists?(Project.build_path(project)) do
+    unless File.exists?(build_path) do
       Logger.info("Performing initial build on new workspace")
+      File.mkdir_p!(build_path)
 
       result =
         RemoteControl.Mix.in_project(project, fn _ ->
-          with_progress "compile", fn ->
+          with_progress building_label(project), fn ->
             Mix.Task.run(:compile, mix_compile_opts(false))
           end
         end)
 
       case result do
-        {:error, {:exception, ex}} ->
+        {:error, {:exception, ex}, _} ->
           Logger.error("Initial compile failed #{Exception.message(ex)}")
 
-        _ ->
-          Logger.info("initial build complete")
+        other ->
+          Logger.info("initial build complete #{inspect(other)}")
       end
     end
 
@@ -69,7 +71,7 @@ defmodule Lexical.RemoteControl.Build.State do
     project = state.project
 
     Build.with_lock(fn ->
-      {elapsed_us, result} = :timer.tc(fn -> safe_compile_project(force?) end)
+      {elapsed_us, result} = :timer.tc(fn -> safe_compile_project(state.project, force?) end)
       elapsed_ms = to_ms(elapsed_us)
 
       {compile_message, diagnostics} =
@@ -153,7 +155,7 @@ defmodule Lexical.RemoteControl.Build.State do
   def set_compiler_options do
     Code.compiler_options(
       parser_options: parser_options(),
-      tracers: [RemoteControl.CompileTracer]
+      tracers: [RemoteControl.Compilation.Tracer]
     )
 
     :ok
@@ -176,6 +178,10 @@ defmodule Lexical.RemoteControl.Build.State do
     end
   end
 
+  def building_label(%Project{} = project) do
+    "Building #{Project.name(project)}"
+  end
+
   defp now do
     System.system_time(:millisecond)
   end
@@ -185,7 +191,7 @@ defmodule Lexical.RemoteControl.Build.State do
     millis_since_last_edit >= edit_window_millis()
   end
 
-  defp safe_compile_project(force?) do
+  defp safe_compile_project(%Project{} = project, force?) do
     RemoteControl.Mix.in_project(fn _ ->
       Mix.Task.clear()
 
@@ -194,7 +200,7 @@ defmodule Lexical.RemoteControl.Build.State do
       compile_fun = fn ->
         Mix.Task.clear()
 
-        with_progress "compile", fn ->
+        with_progress building_label(project), fn ->
           Mix.Task.run("compile", mix_compile_opts(force?))
         end
       end
@@ -221,22 +227,22 @@ defmodule Lexical.RemoteControl.Build.State do
 
   defp prepare_for_project_build(true = _force?) do
     if connected_to_internet?() do
-      with_progress "local.hex", fn ->
+      with_progress "mix local.hex", fn ->
         Mix.Task.run("local.hex", ~w(--force --if-missing))
       end
 
-      with_progress "local.rebar", fn ->
+      with_progress "mix local.rebar", fn ->
         Mix.Task.run("local.rebar", ~w(--force --if-missing))
       end
 
-      with_progress "deps.get", fn ->
+      with_progress "mix deps.get", fn ->
         Mix.Task.run("deps.get")
       end
     else
       Logger.warn("Could not connect to hex.pm, dependencies will not be fetched")
     end
 
-    with_progress "deps.compile", fn ->
+    with_progress "mix deps.compile", fn ->
       Mix.Task.run("deps.safe_compile", ~w(--skip-umbrella-children))
     end
 

apps/remote_control/lib/lexical/remote_control/code_mod/format.ex

@@ -70,26 +70,71 @@ defmodule Lexical.RemoteControl.CodeMod.Format do
   end
 
   defp formatter_for_file(%Project{} = project, file_path) do
-    fetch_formatter = fn -> Mix.Tasks.Format.formatter_for_file(file_path) end
+    fetch_formatter = fn _ -> Mix.Tasks.Format.formatter_for_file(file_path) end
 
     {formatter, _opts} =
       if RemoteControl.project_node?() do
-        case RemoteControl.Mix.in_project(project, fn _ -> fetch_formatter.() end) do
+        case RemoteControl.Mix.in_project(project, fetch_formatter) do
           {:ok, result} ->
             result
 
-          _ ->
+          _error ->
+            formatter_opts =
+              case find_formatter_exs(project, file_path) do
+                {:ok, opts} ->
+                  opts
+
+                :error ->
+                  Logger.warn("Could not find formatter options for file #{file_path}")
+                  []
+              end
+
             formatter = fn source ->
-              formatted_source = Code.format_string!(source)
+              formatted_source = Code.format_string!(source, formatter_opts)
               IO.iodata_to_binary([formatted_source, ?\n])
             end
 
             {formatter, nil}
         end
       else
-        fetch_formatter.()
+        fetch_formatter.(nil)
       end
 
     formatter
   end
+
+  defp find_formatter_exs(%Project{} = project, file_path) do
+    root_dir = Project.root_path(project)
+    do_find_formatter_exs(root_dir, file_path)
+  end
+
+  defp do_find_formatter_exs(root_path, root_path) do
+    formatter_exs_contents(root_path)
+  end
+
+  defp do_find_formatter_exs(root_path, current_path) do
+    with :error <- formatter_exs_contents(current_path) do
+      parent =
+        current_path
+        |> Path.join("..")
+        |> Path.expand()
+
+      do_find_formatter_exs(root_path, parent)
+    end
+  end
+
+  defp formatter_exs_contents(current_path) do
+    formatter_exs = Path.join(current_path, ".formatter.exs")
+
+    with true <- File.exists?(formatter_exs),
+         {formatter_terms, _binding} <- Code.eval_file(formatter_exs) do
+      Logger.info("found formatter in #{current_path}")
+      {:ok, formatter_terms}
+    else
+      err ->
+        Logger.info("No formatter found in #{current_path} error was #{inspect(err)}")
+
+        :error
+    end
+  end
 end

apps/remote_control/lib/lexical/remote_control/compilation/dispatch.ex

@@ -0,0 +1,54 @@
+defmodule Lexical.RemoteControl.Compilation.Dispatch do
+  alias Lexical.RemoteControl
+  alias Lexical.RemoteControl.Api.Messages
+  alias Lexical.RemoteControl.Build
+  alias Lexical.RemoteControl.ModuleMappings
+  alias Lexical.RemoteControl.Plugin
+
+  import Messages
+  use GenServer
+
+  def dispatch(module_updated() = message) do
+    GenServer.cast(__MODULE__, {:dispatch, message})
+  end
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  def init(_) do
+    {:ok, nil}
+  end
+
+  def handle_cast({:dispatch, module_updated() = message}, state) do
+    module_updated(name: module_name, file: filename) = message
+    ModuleMappings.update(module_name, filename)
+    RemoteControl.notify_listener(message)
+    Plugin.on_module_updated(module_name)
+    maybe_report_progress(filename)
+    {:noreply, state}
+  end
+
+  defp maybe_report_progress(file) do
+    if Path.extname(file) == ".ex" do
+      file
+      |> progress_message()
+      |> RemoteControl.notify_listener()
+    end
+  end
+
+  defp progress_message(file) do
+    relative_path_elements =
+      file
+      |> Path.relative_to_cwd()
+      |> Path.split()
+
+    base_dir = List.first(relative_path_elements)
+    file_name = List.last(relative_path_elements)
+
+    message = "compiling: " <> Path.join([base_dir, "...", file_name])
+
+    label = Build.State.building_label(RemoteControl.get_project())
+    project_progress(label: label, message: message)
+  end
+end