perf: avoid forced warm-start compiles

apps/engine/lib/engine/build/project.ex

@@ -9,23 +9,23 @@ defmodule Engine.Build.Project do
 
   require Logger
 
-  def compile(%Project{kind: :mix} = project, initial?) do
+  def compile(%Project{kind: :mix} = project, initial?, force?) do
     Engine.Mix.in_project(fn _ ->
       Logger.info("Building #{Project.display_name(project)}")
 
       Progress.with_progress("Building #{Project.display_name(project)}", fn token ->
         Build.set_progress_token(token)
 
         try do
-          {:done, do_compile(project, initial?, token)}
+          {:done, do_compile(project, initial?, force?, token)}
         after
           Build.clear_progress_token()
         end
       end)
     end)
   end
 
-  def compile(%Project{}, _initial?) do
+  def compile(%Project{}, _initial?, _force?) do
     :ok
   end
 
@@ -54,7 +54,7 @@ defmodule Engine.Build.Project do
     :ok
   end
 
-  defp do_compile(project, initial?, token) do
+  defp do_compile(project, initial?, force?, token) do
     Mix.Task.clear()
 
     if initial? do
@@ -66,7 +66,7 @@ defmodule Engine.Build.Project do
     compile_fun = fn ->
       Mix.Task.clear()
       Progress.report(token, message: "Compiling #{Project.display_name(project)}")
-      result = compile_in_isolation(initial?)
+      result = compile_in_isolation(force?)
       maybe_load_modules()
       Engine.Mix.ensure_hex_and_rebar()
       Mix.Task.run(:loadpaths)
@@ -105,10 +105,10 @@ defmodule Engine.Build.Project do
     end
   end
 
-  defp compile_in_isolation(initial?) do
+  defp compile_in_isolation(force?) do
     compile_fun = fn ->
       Engine.Mix.ensure_hex_and_rebar()
-      Mix.Task.run(:compile, Build.State.mix_compile_opts(initial?))
+      Mix.Task.run(:compile, Build.State.mix_compile_opts(force?))
     end
 
     case Isolation.invoke(compile_fun) do

apps/engine/lib/engine/build/state.ex

@@ -14,6 +14,7 @@ defmodule Engine.Build.State do
             build_number: 0,
             uri_to_document: %{},
             project_compile: :none,
+            initial_compile?: true,
             last_deps_fetch_result: nil
 
   def new(%Project{} = project) do
@@ -32,7 +33,7 @@ defmodule Engine.Build.State do
     # compiled because they might have unsaved changes, and we want that state
     # to be the latest state of the project.
     new_state =
-      Enum.reduce(new_state.uri_to_document, state, fn {_uri, document}, state ->
+      Enum.reduce(new_state.uri_to_document, new_state, fn {_uri, document}, state ->
         compile_file(state, document)
       end)
 
@@ -118,7 +119,7 @@ defmodule Engine.Build.State do
   defp normalize_fetch_deps_result({:ok, :ok}), do: :ok
   defp normalize_fetch_deps_result(result), do: result
 
-  defp compile_project(%__MODULE__{} = state, initial?) do
+  defp compile_project(%__MODULE__{} = state, force?) do
     state = increment_build_number(state)
     project = state.project
 
@@ -128,7 +129,10 @@ defmodule Engine.Build.State do
 
       Engine.broadcast(compile_requested_message)
       Engine.Compilation.TraceBuffer.discard()
-      {elapsed_us, result} = :timer.tc(fn -> Build.Project.compile(project, initial?) end)
+
+      {elapsed_us, result} =
+        :timer.tc(fn -> Build.Project.compile(project, state.initial_compile?, force?) end)
+
       elapsed_ms = to_ms(elapsed_us)
 
       {compile_message, diagnostics} =
@@ -171,7 +175,7 @@ defmodule Engine.Build.State do
       Engine.broadcast(diagnostics_message)
     end)
 
-    state
+    %__MODULE__{state | initial_compile?: false}
   end
 
   def compile_file(%__MODULE__{} = state, %Document{} = document) do
@@ -244,7 +248,7 @@ defmodule Engine.Build.State do
     :ok
   end
 
-  def mix_compile_opts(initial?) do
+  def mix_compile_opts(force?) do
     opts = ~w(
         --return-errors
         --ignore-module-conflict
@@ -255,7 +259,7 @@ defmodule Engine.Build.State do
         --no-prune-code-paths
     )
 
-    if initial? do
+    if force? do
       ["--force" | opts]
     else
       opts

apps/engine/test/engine/build/state_test.exs

@@ -107,7 +107,7 @@ defmodule Engine.Build.StateTest do
       State.on_file_compile(state, document)
 
       refute_called(Build.Document.compile(document))
-      refute_called(Build.Project.compile(_, _))
+      refute_called(Build.Project.compile(_, _, _))
     end
 
     test "it compiles files when on_timeout is called", %{state: state, document: document} do
@@ -116,7 +116,7 @@ defmodule Engine.Build.StateTest do
       |> State.on_timeout()
 
       assert_called(Build.Document.compile(document))
-      refute_called(Build.Project.compile(_, _))
+      refute_called(Build.Project.compile(_, _, _))
     end
   end
 
@@ -125,28 +125,42 @@ defmodule Engine.Build.StateTest do
 
     test "doesn't compile immediately if forced", %{state: state} do
       State.on_project_compile(state, true)
-      refute_called(Build.Project.compile(_, _))
+      refute_called(Build.Project.compile(_, _, _))
     end
 
     test "doesn't compile immediately", %{state: state} do
       State.on_project_compile(state, false)
-      refute_called(Build.Project.compile(_, _))
+      refute_called(Build.Project.compile(_, _, _))
     end
 
     test "compiles if force is true after on_timeout is called", %{state: state} do
       state
       |> State.on_project_compile(true)
       |> State.on_timeout()
 
-      assert_called(Build.Project.compile(_, true))
+      assert_called(Build.Project.compile(_, true, true))
     end
 
     test "compiles after on_timeout is called", %{state: state} do
       state
       |> State.on_project_compile(false)
       |> State.on_timeout()
 
-      assert_called(Build.Project.compile(_, false))
+      assert_called(Build.Project.compile(_, true, false))
+    end
+
+    test "prepares the project only during the first compile", %{state: state} do
+      state =
+        state
+        |> State.on_project_compile(false)
+        |> State.on_timeout()
+
+      state
+      |> State.on_project_compile(true)
+      |> State.on_timeout()
+
+      assert_called(Build.Project.compile(_, true, false), 1)
+      assert_called(Build.Project.compile(_, false, true), 1)
     end
   end
 
@@ -162,7 +176,7 @@ defmodule Engine.Build.StateTest do
       |> State.on_file_compile(document)
 
       refute_called(Build.Document.compile(_))
-      refute_called(Build.Project.compile(_, _))
+      refute_called(Build.Project.compile(_, _, _))
     end
 
     test "compiles when on_timeout is called if both documents and projects are added", %{
@@ -175,7 +189,7 @@ defmodule Engine.Build.StateTest do
       |> State.on_timeout()
 
       assert_called(Build.Document.compile(_))
-      assert_called(Build.Project.compile(_, _))
+      assert_called(Build.Project.compile(_, _, _))
     end
   end
 
@@ -196,7 +210,7 @@ defmodule Engine.Build.StateTest do
     test "project compilation returns :ok without calling Mix", %{state: state} do
       patch(Engine.Mix, :in_project, fn _project, _fun -> {:error, :should_not_be_called} end)
 
-      assert Engine.Build.Project.compile(state.project, true) == :ok
+      assert Engine.Build.Project.compile(state.project, true, false) == :ok
 
       refute_called(Engine.Mix.in_project(_, _))
     end

apps/expert/lib/expert/engine_node.ex

@@ -339,6 +339,7 @@ defmodule Expert.EngineNode do
   @impl true
   def handle_info({:DOWN, _ref, :process, _object, _reason}, %State{} = state) do
     state = State.on_monitored_dead(state)
+    State.maybe_reply_to_stopper(state)
     {:stop, :shutdown, state}
   end
 

apps/expert/lib/expert/project/indexer.ex

@@ -85,7 +85,8 @@ defmodule Expert.Project.Indexer do
 
   @impl GenServer
   def handle_continue(:maybe_initial_compile, %State{initial_compile?: true} = state) do
-    Node.trigger_build(state.project)
+    force? = Search.Store.load_status(state.project) not in [:stale, :ready]
+    Node.trigger_build(state.project, force?)
     {:noreply, state}
   end
 

apps/expert/lib/expert/project/node.ex

@@ -41,10 +41,10 @@ defmodule Expert.Project.Node do
     |> GenServer.call(:node_name)
   end
 
-  def trigger_build(%Project{} = project) do
+  def trigger_build(%Project{} = project, force? \\ true) do
     project
     |> name()
-    |> GenServer.cast(:trigger_build)
+    |> GenServer.cast({:trigger_build, force?})
   end
 
   @impl GenServer
@@ -77,8 +77,8 @@ defmodule Expert.Project.Node do
   end
 
   @impl GenServer
-  def handle_cast(:trigger_build, %State{} = state) do
-    EngineApi.schedule_compile(state.project, true)
+  def handle_cast({:trigger_build, force?}, %State{} = state) do
+    EngineApi.schedule_compile(state.project, force?)
     {:noreply, state}
   end
 

apps/expert/test/expert/engine_node_test.exs

@@ -28,6 +28,26 @@ defmodule Expert.EngineNodeTest do
     assert_eventually(Process.whereis(EngineNode.name(project)) == nil, :timer.seconds(5))
   end
 
+  test "replies to a pending stopper when the monitored process dies", %{project: project} do
+    reply_ref = make_ref()
+
+    state = %{
+      EngineNode.State.new(project)
+      | status: :stopping,
+        stopped_by: {self(), reply_ref}
+    }
+
+    patch(EngineNode.State, :on_monitored_dead, fn _state -> state end)
+
+    assert {:stop, :shutdown, ^state} =
+             EngineNode.handle_info(
+               {:DOWN, make_ref(), :process, self(), :shutdown},
+               state
+             )
+
+    assert_receive {^reply_ref, :ok}
+  end
+
   test "it should be stopped atomically when the startup process is dead", %{project: project} do
     test_pid = self()
 

apps/expert/test/expert/project/indexer_test.exs

@@ -9,6 +9,7 @@ defmodule Expert.Project.IndexerTest do
 
   alias Expert.EngineApi
   alias Expert.Project.Indexer
+  alias Expert.Project.Node, as: ProjectNode
   alias Expert.Search.Store
   alias Expert.Search.Store.Backends.Sqlite
   alias Expert.Test.DispatchFake
@@ -38,6 +39,46 @@ defmodule Expert.Project.IndexerTest do
     {:ok, project: project, task_supervisor: task_supervisor}
   end
 
+  describe "initial project compile" do
+    test "forces compilation when the persisted index is empty", %{
+      project: project,
+      task_supervisor: task_supervisor
+    } do
+      test_pid = self()
+
+      patch(Store, :load_status, fn ^project -> :empty end)
+
+      patch(ProjectNode, :trigger_build, fn ^project, force? ->
+        send(test_pid, {:trigger_build, force?})
+      end)
+
+      start_supervised!(
+        {Indexer, [project, task_supervisor: task_supervisor, initial_compile?: true]}
+      )
+
+      assert_receive {:trigger_build, true}
+    end
+
+    test "uses a normal compilation when a persisted index exists", %{
+      project: project,
+      task_supervisor: task_supervisor
+    } do
+      test_pid = self()
+
+      patch(Store, :load_status, fn ^project -> :stale end)
+
+      patch(ProjectNode, :trigger_build, fn ^project, force? ->
+        send(test_pid, {:trigger_build, force?})
+      end)
+
+      start_supervised!(
+        {Indexer, [project, task_supervisor: task_supervisor, initial_compile?: true]}
+      )
+
+      assert_receive {:trigger_build, false}
+    end
+  end
+
   test "creates the initial index after a successful project compile", %{
     project: project,
     task_supervisor: task_supervisor

apps/expert/test/expert/project/node_test.exs

@@ -1,6 +1,7 @@
 defmodule Expert.Project.NodeTest do
   use ExUnit.Case
   use Forge.Test.EventualAssertions
+  use Patch
 
   import Forge.EngineApi.Messages
   import Forge.Test.Fixtures
@@ -29,6 +30,18 @@ defmodule Expert.Project.NodeTest do
     assert_receive project_compiled(), :timer.seconds(15)
   end
 
+  test "trigger_build forwards the requested compile mode", %{project: project} do
+    test_pid = self()
+
+    patch(EngineApi, :schedule_compile, fn ^project, force? ->
+      send(test_pid, {:schedule_compile, force?})
+    end)
+
+    EngineNode.trigger_build(project, false)
+
+    assert_receive {:schedule_compile, false}
+  end
+
   test "remote control is started when the node starts", %{project: project} do
     apps = EngineApi.call(project, Application, :started_applications)
     app_names = Enum.map(apps, &elem(&1, 0))