diff --git a/apps/expert/lib/expert/code_intelligence/references.ex b/apps/expert/lib/expert/code_intelligence/references.ex index 5d6a00c80..63ef022ee 100644 --- a/apps/expert/lib/expert/code_intelligence/references.ex +++ b/apps/expert/lib/expert/code_intelligence/references.ex @@ -21,7 +21,8 @@ defmodule Expert.CodeIntelligence.References do %Project{} = project, %Analysis{} = analysis, %Position{} = position, - include_definitions? + include_definitions?, + engine_ready? \\ true ) do case variable_at(analysis, position) do {:ok, name} -> @@ -34,8 +35,11 @@ defmodule Expert.CodeIntelligence.References do [_ | _] = references -> references - [] -> + [] when engine_ready? -> engine_references(project, analysis, position, include_definitions?) + + [] -> + [] end end end diff --git a/apps/expert/lib/expert/project/indexer.ex b/apps/expert/lib/expert/project/indexer.ex index 383e57658..0006540e9 100644 --- a/apps/expert/lib/expert/project/indexer.ex +++ b/apps/expert/lib/expert/project/indexer.ex @@ -8,7 +8,6 @@ defmodule Expert.Project.Indexer do import Forge.EngineApi.Messages alias Expert.EngineApi - alias Expert.Project.Node alias Expert.Search alias Forge.Project @@ -21,33 +20,28 @@ defmodule Expert.Project.Indexer do :task_supervisor, :create_index, :update_index, - :initial_compile?, pending?: false ] - def new(%Project{} = project, opts) do + def new(project, opts) do %__MODULE__{ project: project, task_supervisor: Keyword.fetch!(opts, :task_supervisor), create_index: Keyword.fetch!(opts, :create_index), - update_index: Keyword.fetch!(opts, :update_index), - initial_compile?: Keyword.get(opts, :initial_compile?, false) + update_index: Keyword.fetch!(opts, :update_index) } end end - def start_link(%Project{} = project) do - start_link(project, []) - end + def start_link(%Project{} = project), do: start_link(project, []) - def start_link(%Project{} = project, opts) when is_list(opts) do + def start_link(%Project{} = project, opts) do opts = Keyword.merge( [ task_supervisor: task_supervisor_name(project), create_index: &Search.Indexer.create_index/1, - update_index: &Search.Indexer.update_index/1, - initial_compile?: false + update_index: &Search.Indexer.update_index/1 ], opts ) @@ -55,14 +49,9 @@ defmodule Expert.Project.Indexer do GenServer.start_link(__MODULE__, [project, opts], name: name(project)) end - def child_spec(%Project{} = project) do - %{ - id: {__MODULE__, Project.unique_name(project)}, - start: {__MODULE__, :start_link, [project]} - } - end + def child_spec(%Project{} = project), do: child_spec([project]) - def child_spec([%Project{} = project | opts]) when is_list(opts) do + def child_spec([%Project{} = project | opts]) do %{ id: {__MODULE__, Project.unique_name(project)}, start: {__MODULE__, :start_link, [project, opts]} @@ -75,107 +64,102 @@ defmodule Expert.Project.Indexer do :"#{Project.unique_name(project)}::indexer_task_supervisor" end + def refresh(%Project{} = project), do: GenServer.cast(name(project), :refresh) + @impl GenServer - def init([%Project{} = project, opts]) do - EngineApi.register_listener(project, self(), [project_compiled()]) - {:ok, State.new(project, opts), {:continue, :maybe_initial_compile}} + def init([project, opts]) do + Process.flag(:trap_exit, true) + {:ok, State.new(project, opts), {:continue, :warmup}} end @impl GenServer - def handle_continue(:maybe_initial_compile, %State{initial_compile?: true} = state) do - force? = Search.Store.load_status(state.project) not in [:stale, :ready] - Node.trigger_build(state.project, force?) - {:noreply, state} - end + def handle_continue(:warmup, %State{} = state) do + task = + Task.Supervisor.async(state.task_supervisor, fn -> + with :ok <- Search.Store.enable(state.project), + :empty <- Search.Store.load_status(state.project) do + Search.Indexer.warmup(state.project) + else + status when status in [:stale, :ready] -> :ok + error -> error + end + end) - def handle_continue(:maybe_initial_compile, %State{} = state), do: {:noreply, state} + {:noreply, %State{state | task: {:warmup, task}}} + end @impl GenServer - def handle_info(project_compiled(status: status), %State{} = state) - when status in [:success, :successful, :error] do - {:noreply, start_or_queue_index(state)} + def handle_cast(:refresh, %State{task: {:warmup, task}} = state) do + Task.shutdown(task, :brutal_kill) + {:noreply, start_refresh(%State{state | task: nil})} end - def handle_info({ref, result}, %State{task: %Task{ref: ref}} = state) do - Process.demonitor(ref, [:flush]) - log_index_result(result) + def handle_cast(:refresh, %State{task: nil} = state) do + {:noreply, start_refresh(state)} + end - {:noreply, complete_index(state, result)} + def handle_cast(:refresh, %State{} = state) do + {:noreply, %State{state | pending?: true}} + end + + @impl GenServer + def handle_info({ref, result}, %State{task: {kind, %Task{ref: ref}}} = state) do + Process.demonitor(ref, [:flush]) + log_result(kind, result) + {:noreply, finish_task(state, result)} end def handle_info( {:DOWN, ref, :process, _pid, reason}, - %State{task: %Task{ref: ref}} = state + %State{task: {kind, %Task{ref: ref}}} = state ) do - Logger.error("Search indexing failed: #{Exception.format_exit(reason)}") - - {:noreply, complete_index(state, {:error, reason})} + Logger.error("Index #{kind} failed: #{Exception.format_exit(reason)}") + {:noreply, finish_task(state, {:error, reason})} end - def handle_info(_message, %State{} = state), do: {:noreply, state} + def handle_info({:EXIT, _pid, _reason}, state), do: {:noreply, state} - defp start_or_queue_index(%State{task: %Task{}} = state), do: %State{state | pending?: true} + @impl GenServer + def terminate(_reason, %State{task: {_kind, task}}), do: Task.shutdown(task, :brutal_kill) + def terminate(_reason, _state), do: :ok - defp start_or_queue_index(%State{} = state) do + defp start_refresh(%State{} = state) do task = - Task.Supervisor.async_nolink(state.task_supervisor, fn -> - run_index(state.project, state.create_index, state.update_index) + Task.Supervisor.async(state.task_supervisor, fn -> + with :ok <- Search.Store.enable(state.project) do + case Search.Store.load_status(state.project) do + :empty -> state.create_index.(state.project) + _ -> update_index(state) + end + end end) - %State{state | task: task, pending?: false} + %State{state | task: {:refresh, task}, pending?: false} end - defp complete_index(%State{pending?: true} = state, _result), - do: start_or_queue_index(%State{state | task: nil, pending?: false}) - - defp complete_index(%State{} = state, :ok) do - EngineApi.broadcast(state.project, project_index_ready(project: state.project)) - %State{state | task: nil} - end - - defp complete_index(%State{} = state, _result) do - %State{state | task: nil} - end - - defp run_index(%Project{} = project, create_index, update_index) do - with :ok <- Search.Store.enable(project) do - persist_index(project, Search.Store.load_status(project), create_index, update_index) - end - end - - defp persist_index(%Project{} = project, :empty, create_index, _update_index) do - persist_full_index(project, create_index) - end - - defp persist_index(%Project{} = project, _status, create_index, update_index) do - persist_incremental_index(project, create_index, update_index) - end - - defp persist_full_index(%Project{} = project, create_index) do - create_index.(project) - end - - defp persist_incremental_index(%Project{} = project, create_index, update_index) do - case update_index.(project) do + defp update_index(%State{} = state) do + case state.update_index.(state.project) do {:error, {:store, reason}} -> Logger.warning( "Could not persist incremental index update, rebuilding full index: #{inspect(reason)}" ) - persist_index(project, :empty, create_index, update_index) + state.create_index.(state.project) result -> result end end - defp log_index_result(:ok), do: :ok + defp finish_task(%State{pending?: true} = state, _result), do: start_refresh(state) - defp log_index_result({:error, reason}) do - Logger.warning("Could not refresh index: #{inspect(reason)}") + defp finish_task(%State{task: {:refresh, _}} = state, :ok) do + EngineApi.broadcast(state.project, project_index_ready(project: state.project)) + %State{state | task: nil} end - defp log_index_result(other) do - Logger.warning("Unexpected index refresh result: #{inspect(other)}") - end + defp finish_task(%State{} = state, _result), do: %State{state | task: nil} + + defp log_result(kind, :ok), do: Logger.info("Index #{kind} finished") + defp log_result(kind, result), do: Logger.warning("Index #{kind} returned #{inspect(result)}") end diff --git a/apps/expert/lib/expert/project/mix_project.ex b/apps/expert/lib/expert/project/mix_project.ex new file mode 100644 index 000000000..73c4f0a13 --- /dev/null +++ b/apps/expert/lib/expert/project/mix_project.ex @@ -0,0 +1,38 @@ +defmodule Expert.Project.MixProject do + @moduledoc false + + alias Forge.Project + + def configuration(%Project{kind: :mix} = project), do: run(project, []) + def runtime_versions(%Project{} = project), do: run(project, ["--runtime-only"]) + + defp run(project, args) do + script = Path.join(:code.priv_dir(:expert), "read_mix_configuration.exs") + + case Expert.Port.open_elixir(project, args: [script | args]) do + port when is_port(port) -> read_result(port, []) + error -> error + end + end + + defp read_result(port, output) do + receive do + {^port, {:data, data}} -> + read_result(port, [data | output]) + + {^port, {:exit_status, status}} -> + output = output |> Enum.reverse() |> IO.iodata_to_binary() + decode_result(output, status) + end + end + + defp decode_result(output, 0) do + encoded = output |> String.trim() |> String.split("\n") |> List.last() + + with {:ok, binary} <- Base.decode64(encoded) do + :erlang.binary_to_term(binary) + end + end + + defp decode_result(output, status), do: {:error, {:mix_configuration, status, output}} +end diff --git a/apps/expert/lib/expert/project/search_listener.ex b/apps/expert/lib/expert/project/search_listener.ex index 9fe890f01..f040bbaf9 100644 --- a/apps/expert/lib/expert/project/search_listener.ex +++ b/apps/expert/lib/expert/project/search_listener.ex @@ -4,6 +4,8 @@ defmodule Expert.Project.SearchListener do import Forge.EngineApi.Messages alias Expert.EngineApi + alias Expert.Project.Indexer + alias Expert.Project.Node alias Expert.Protocol.Id alias Forge.Formats alias Forge.Project @@ -15,22 +17,35 @@ defmodule Expert.Project.SearchListener do GenServer.start_link(__MODULE__, [project], name: name(project)) end - defp name(%Project{} = project) do + def name(%Project{} = project) do :"#{Project.unique_name(project)}::search_listener" end @impl GenServer def init([%Project{} = project]) do EngineApi.register_listener(project, self(), [ + project_compiled(), project_reindex_requested(), project_reindexed(), search_store_loading() ]) - {:ok, project} + {:ok, project, {:continue, :compile}} end @impl GenServer + def handle_continue(:compile, project) do + Node.trigger_build(project, false) + {:noreply, project} + end + + @impl GenServer + def handle_info(project_compiled(status: status), %Project{} = project) + when status in [:success, :successful, :error] do + Indexer.refresh(project) + {:noreply, project} + end + def handle_info(project_reindex_requested(), %Project{} = project) do Logger.info("project reindex requested") GenLSP.request(Expert.get_lsp(), %Requests.WorkspaceCodeLensRefresh{id: Id.next()}) diff --git a/apps/expert/lib/expert/project/supervisor.ex b/apps/expert/lib/expert/project/supervisor.ex index b866a7a98..846a0fa52 100644 --- a/apps/expert/lib/expert/project/supervisor.ex +++ b/apps/expert/lib/expert/project/supervisor.ex @@ -22,16 +22,16 @@ defmodule Expert.Project.Supervisor do def init(%Project{} = project) do children = [ {EngineSupervisor, project}, - {Node, project}, {Search.Store.backend(), project}, {Search.Store, [project]}, {ModuleRegistry, project}, + {Task.Supervisor, name: Indexer.task_supervisor_name(project)}, + {Indexer, project}, + {Node, project}, {Diagnostics, project}, {Intelligence, project}, - {SearchListener, project}, {Reindex, project}, - {Task.Supervisor, name: Indexer.task_supervisor_name(project)}, - {Indexer, [project, initial_compile?: true]} + {SearchListener, project} ] Supervisor.init(children, strategy: :rest_for_one) diff --git a/apps/expert/lib/expert/provider/handlers/document_symbols.ex b/apps/expert/lib/expert/provider/handlers/document_symbols.ex index e4f7557d8..edaed8e01 100644 --- a/apps/expert/lib/expert/provider/handlers/document_symbols.ex +++ b/apps/expert/lib/expert/provider/handlers/document_symbols.ex @@ -9,6 +9,9 @@ defmodule Expert.Provider.Handlers.DocumentSymbols do alias GenLSP.Requests alias GenLSP.Structures + @impl Expert.Provider.Handler + def requires_engine?, do: false + @impl Expert.Provider.Handler def handle(%Requests.TextDocumentDocumentSymbol{}, %Context{} = context) do %Context{document: document, project: project} = context diff --git a/apps/expert/lib/expert/provider/handlers/find_references.ex b/apps/expert/lib/expert/provider/handlers/find_references.ex index 5d469669d..97f950bad 100644 --- a/apps/expert/lib/expert/provider/handlers/find_references.ex +++ b/apps/expert/lib/expert/provider/handlers/find_references.ex @@ -3,11 +3,15 @@ defmodule Expert.Provider.Handlers.FindReferences do alias Expert.CodeIntelligence.References alias Expert.Document.Context + alias Expert.Project.Store alias Forge.Ast alias Forge.Document alias GenLSP.Requests.TextDocumentReferences alias GenLSP.Structures + @impl Expert.Provider.Handler + def requires_engine?, do: false + @impl Expert.Provider.Handler def handle( %TextDocumentReferences{params: %Structures.ReferenceParams{} = params}, @@ -19,7 +23,13 @@ defmodule Expert.Provider.Handlers.FindReferences do locations = case Document.Store.fetch(document.uri, :analysis) do {:ok, _document, %Ast.Analysis{} = analysis} -> - References.references(project, analysis, params.position, include_declaration?) + References.references( + project, + analysis, + params.position, + include_declaration?, + Store.ready?(project) + ) _ -> nil diff --git a/apps/expert/lib/expert/provider/handlers/go_to_definition.ex b/apps/expert/lib/expert/provider/handlers/go_to_definition.ex index 169139888..a7ce34cd8 100644 --- a/apps/expert/lib/expert/provider/handlers/go_to_definition.ex +++ b/apps/expert/lib/expert/provider/handlers/go_to_definition.ex @@ -4,11 +4,15 @@ defmodule Expert.Provider.Handlers.GoToDefinition do alias Expert.CodeIntelligence.Definition alias Expert.Document.Context alias Expert.EngineApi + alias Expert.Project.Store alias GenLSP.Requests alias GenLSP.Structures require Logger + @impl Expert.Provider.Handler + def requires_engine?, do: false + @impl Expert.Provider.Handler def handle( %Requests.TextDocumentDefinition{params: %Structures.DefinitionParams{} = params}, @@ -18,7 +22,7 @@ defmodule Expert.Provider.Handlers.GoToDefinition do result = case Definition.definition(project, document, params.position) do - {:ok, nil} -> EngineApi.definition(project, document, params.position) + {:ok, nil} -> fallback_to_engine(project, document, params.position) {:ok, _native_location} = result -> result end @@ -31,4 +35,12 @@ defmodule Expert.Provider.Handlers.GoToDefinition do {:ok, nil} end end + + defp fallback_to_engine(project, document, position) do + if Store.ready?(project) do + EngineApi.definition(project, document, position) + else + {:ok, nil} + end + end end diff --git a/apps/expert/lib/expert/search/indexer.ex b/apps/expert/lib/expert/search/indexer.ex index 2ff8c23d6..6ae31e85d 100644 --- a/apps/expert/lib/expert/search/indexer.ex +++ b/apps/expert/lib/expert/search/indexer.ex @@ -19,6 +19,20 @@ defmodule Expert.Search.Indexer do end end + def warmup(%Project{} = project, opts \\ []) when is_list(opts) do + :ok = ModuleRegistry.clear(project) + paths = Keyword.get_lazy(opts, :paths, fn -> Paths.from_disk(project) end) + + with :ok <- ManifestStore.invalidate(project), + :ok <- store_result(Store.replace(project, [])), + {:ok, state} <- + paths + |> index_stream(project) + |> persist_stream(new_stream_state(), project) do + ManifestStore.commit(project, Manifest.new(manifest_entries(state))) + end + end + def update_index(%Project{} = project, opts \\ []) when is_list(opts) do with path_to_ids when is_map(path_to_ids) <- Store.path_to_ids(project), {:ok, manifest} <- update_index(project, path_to_ids, opts) do diff --git a/apps/expert/lib/expert/search/indexer/paths.ex b/apps/expert/lib/expert/search/indexer/paths.ex index f563e4ad2..71dcbb254 100644 --- a/apps/expert/lib/expert/search/indexer/paths.ex +++ b/apps/expert/lib/expert/search/indexer/paths.ex @@ -1,5 +1,6 @@ defmodule Expert.Search.Indexer.Paths do alias Expert.EngineApi + alias Expert.Project.MixProject alias Forge.Document alias Forge.Project @@ -17,6 +18,9 @@ defmodule Expert.Search.Indexer.Paths do for_project(project, &EngineApi.project_configuration(project, &1)) end + @doc "Discovers source files and existing build output with the project's local Mix configuration." + def from_disk(%Project{} = project), do: for_project(project, &MixProject.configuration/1) + def for_project(%Project{kind: :bare} = project, _configuration) do %__MODULE__{source_paths: source_files(Project.root_path(project), [])} end diff --git a/apps/expert/lib/expert/search/store/backends/sqlite.ex b/apps/expert/lib/expert/search/store/backends/sqlite.ex index 8c8b48ceb..24ff46620 100644 --- a/apps/expert/lib/expert/search/store/backends/sqlite.ex +++ b/apps/expert/lib/expert/search/store/backends/sqlite.ex @@ -3,7 +3,6 @@ defmodule Expert.Search.Store.Backends.Sqlite do use GenServer - alias Expert.EngineApi alias Expert.Search.Store.Backend alias Forge.Document.Position alias Forge.Document.Range @@ -1200,6 +1199,9 @@ defmodule Expert.Search.Store.Backends.Sqlite do do: Entry.is_block(left_entry) == Entry.is_block(right_entry) defp runtime_versions(%Project{} = project, opts) do - Keyword.get_lazy(opts, :runtime_versions, fn -> EngineApi.runtime_versions(project) end) + Keyword.get_lazy(opts, :runtime_versions, fn -> + {:ok, versions} = Expert.Project.MixProject.runtime_versions(project) + versions + end) end end diff --git a/apps/expert/priv/read_mix_configuration.exs b/apps/expert/priv/read_mix_configuration.exs new file mode 100644 index 000000000..bbcad3531 --- /dev/null +++ b/apps/expert/priv/read_mix_configuration.exs @@ -0,0 +1,66 @@ +if "--runtime-only" in System.argv() do + release = to_string(:erlang.system_info(:otp_release)) + version_file = Path.join([:code.root_dir(), "releases", release, "OTP_VERSION"]) + + erlang = + case File.read(version_file) do + {:ok, text} -> + case String.split(text, "\n", trim: true) do + [full] -> full + _ -> release + end + + {:error, _} -> + release + end + + {:ok, %{elixir: System.version(), erlang: erlang}} +else + {:ok, _} = Application.ensure_all_started(:mix) + Mix.env(:test) + + normalize_dependency = fn + {app, requirement, opts} when is_list(opts) -> + {app, requirement, Keyword.take(opts, [:app, :only, :path, :targets])} + + {app, opts} when is_list(opts) -> + {app, Keyword.take(opts, [:app, :only, :path, :targets])} + + dependency -> + dependency + end + + # Project code can fail before its dependencies exist. Return that failure to path discovery. + try do + configuration = + Mix.Project.in_project(:expert_indexer, File.cwd!(), [], fn module -> + config = Mix.Project.config() + + dependency_apps = + try do + {:ok, Mix.Project.deps_apps()} + rescue + error -> {:error, Exception.message(error)} + end + + %{ + config: [app: config[:app], deps: Enum.map(config[:deps] || [], normalize_dependency)], + project_config: Keyword.take(module.project(), [:build_path, :deps_build_path]), + build_path: Mix.Project.build_path(), + deps_path: Mix.Project.deps_path(), + apps_paths: Mix.Project.apps_paths(), + dependency_apps: dependency_apps, + env: Mix.env(), + target: Mix.target(), + build_root: System.get_env("MIX_BUILD_ROOT") + } + end) + + {:ok, configuration} + rescue + error -> {:error, Exception.message(error)} + end +end +|> :erlang.term_to_binary() +|> Base.encode64() +|> IO.puts() diff --git a/apps/expert/test/engine/code_intelligence/definition_test.exs b/apps/expert/test/engine/code_intelligence/definition_test.exs index f33e49737..55200cc6f 100644 --- a/apps/expert/test/engine/code_intelligence/definition_test.exs +++ b/apps/expert/test/engine/code_intelligence/definition_test.exs @@ -94,6 +94,7 @@ defmodule Expert.Engine.CodeIntelligence.DefinitionTest do EngineApi.call(project, Application, :put_env, [:language_server, :elixir_src, elixir_src]) assert_receive project_compiled(), @project_compile_timeout + Indexer.refresh(project) assert_receive project_index_ready(project: ^project), @project_index_timeout %{project: project} diff --git a/apps/expert/test/expert/project/indexer_test.exs b/apps/expert/test/expert/project/indexer_test.exs index 7cd62348c..7581ae56f 100644 --- a/apps/expert/test/expert/project/indexer_test.exs +++ b/apps/expert/test/expert/project/indexer_test.exs @@ -1,126 +1,66 @@ defmodule Expert.Project.IndexerTest do use ExUnit.Case, async: false use Patch - use Expert.Test.DispatchFake import Forge.EngineApi.Messages + import Forge.Test.EventualAssertions import Forge.Test.Fixtures alias Expert.EngineApi alias Expert.Project.Indexer - alias Expert.Project.Node, as: ProjectNode + alias Expert.Search alias Expert.Search.Store alias Expert.Search.Store.Backends.Sqlite - alias Expert.Test.DispatchFake - alias Forge.Project - - # The first assertion after a broadcast waits on `Search.Store.enable/1`, which the store - # allows 30 seconds for. Setup destroys the database first, so a cold rebuild can outlast the - # global `assert_receive_timeout` of one second. - @enable_timeout :timer.seconds(5) setup do project = project() - DispatchFake.start() Sqlite.destroy_all(project) - start_supervised!({Sqlite, [project, runtime_versions: runtime_versions()]}) - start_supervised!({Store, [project, Sqlite]}) - - task_supervisor = :"#{Project.unique_name(project)}::indexer_test_task_supervisor" - start_supervised!({Task.Supervisor, name: task_supervisor}) - - EngineApi.register_listener(project, self(), [project_index_ready()]) + start_supervised!( + {Sqlite, [project, runtime_versions: %{erlang: "engine-erlang", elixir: "engine-elixir"}]} + ) + start_supervised!({Store, [project, Sqlite]}) + start_supervised!({Search.Indexer.ModuleRegistry, project}) + start_supervised!({Task.Supervisor, name: Indexer.task_supervisor_name(project)}) + patch(EngineApi, :call, fn _, _, _, _ -> flunk("Index tasks must not control the Engine") end) + test_pid = self() + patch(EngineApi, :broadcast, fn ^project, message -> send(test_pid, message) end) + patch(Search.Indexer, :warmup, fn _ -> :ok end) on_exit(fn -> Sqlite.destroy_all(project) end) - - {:ok, project: project, task_supervisor: task_supervisor} - end - - describe "initial project compile" do - test "forces compilation when the persisted index is empty", %{ - project: project, - task_supervisor: task_supervisor - } do - test_pid = self() - - patch(Store, :load_status, fn ^project -> :empty end) - - patch(ProjectNode, :trigger_build, fn ^project, force? -> - send(test_pid, {:trigger_build, force?}) - end) - - start_supervised!( - {Indexer, [project, task_supervisor: task_supervisor, initial_compile?: true]} - ) - - assert_receive {:trigger_build, true} - end - - test "uses a normal compilation when a persisted index exists", %{ - project: project, - task_supervisor: task_supervisor - } do - test_pid = self() - - patch(Store, :load_status, fn ^project -> :stale end) - - patch(ProjectNode, :trigger_build, fn ^project, force? -> - send(test_pid, {:trigger_build, force?}) - end) - - start_supervised!( - {Indexer, [project, task_supervisor: task_supervisor, initial_compile?: true]} - ) - - assert_receive {:trigger_build, false} - end + {:ok, project: project} end - test "creates the initial index after a successful project compile", %{ - project: project, - task_supervisor: task_supervisor - } do + test "creates an empty index on request", %{project: project} do test_pid = self() start_supervised!( {Indexer, [ project, - task_supervisor: task_supervisor, create_index: fn ^project -> send(test_pid, :create_index) :ok end, - update_index: fn ^project -> - send(test_pid, :update_index) - :ok - end + update_index: fn _ -> flunk("An empty index needs a full index") end ]} ) - EngineApi.broadcast(project, project_compiled(project: project, status: :success)) + Indexer.refresh(project) - assert_receive :create_index, @enable_timeout - refute_receive :update_index + assert_receive :create_index, 5_000 assert_receive project_index_ready(project: ^project) end - test "creates the initial index even when the project compile reports an error", %{ - project: project, - task_supervisor: task_supervisor - } do + test "updates an existing index on request", %{project: project} do test_pid = self() + patch(Store, :load_status, fn ^project -> :ready end) start_supervised!( {Indexer, [ project, - task_supervisor: task_supervisor, - create_index: fn ^project -> - send(test_pid, :create_index) - :ok - end, + create_index: fn _ -> flunk("An existing index needs an update") end, update_index: fn ^project -> send(test_pid, :update_index) :ok @@ -128,16 +68,13 @@ defmodule Expert.Project.IndexerTest do ]} ) - EngineApi.broadcast(project, project_compiled(project: project, status: :error)) + Indexer.refresh(project) - assert_receive :create_index, @enable_timeout + assert_receive :update_index, 5_000 assert_receive project_index_ready(project: ^project) end - test "updates an existing index after later successful project compiles", %{ - project: project, - task_supervisor: task_supervisor - } do + test "rebuilds after an incremental Store write fails", %{project: project} do test_pid = self() patch(Store, :load_status, fn ^project -> :ready end) @@ -145,54 +82,141 @@ defmodule Expert.Project.IndexerTest do {Indexer, [ project, - task_supervisor: task_supervisor, create_index: fn ^project -> send(test_pid, :create_index) :ok end, update_index: fn ^project -> send(test_pid, :update_index) - :ok + {:error, {:store, :write_failed}} end ]} ) - EngineApi.broadcast(project, project_compiled(project: project, status: :success)) + Indexer.refresh(project) - assert_receive :update_index, @enable_timeout - refute_receive :create_index + assert_receive :update_index, 5_000 + assert_receive :create_index assert_receive project_index_ready(project: ^project) end - test "rebuilds after an incremental Store write fails", %{ - project: project, - task_supervisor: task_supervisor - } do + test "a refresh cancels unfinished warmup and keeps the project registry", %{project: project} do test_pid = self() - patch(Store, :load_status, fn ^project -> :ready end) + + patch(Search.Indexer, :warmup, fn ^project -> + registry = Search.Indexer.ModuleRegistry.name(project) + send(test_pid, {:warmup, self(), registry}) + Process.sleep(:infinity) + end) start_supervised!( {Indexer, [ project, - task_supervisor: task_supervisor, create_index: fn ^project -> send(test_pid, :create_index) :ok - end, - update_index: fn ^project -> - send(test_pid, :update_index) - {:error, {:store, :write_failed}} end ]} ) - EngineApi.broadcast(project, project_compiled(project: project, status: :success)) + assert_receive {:warmup, pid, registry}, 5_000 + ref = Process.monitor(pid) + Indexer.refresh(project) - assert_receive :update_index, @enable_timeout + assert_receive {:DOWN, ^ref, :process, ^pid, :killed} assert_receive :create_index + assert :ets.info(registry, :owner) == Process.whereis(registry) + assert_receive project_index_ready(project: ^project) + end + + test "completed warmup permits a later refresh", %{project: project} do + test_pid = self() + + patch(Search.Indexer, :warmup, fn ^project -> + send(test_pid, {:warmup, self()}) + :ok + end) + + start_supervised!({Indexer, [project, create_index: fn _ -> :ok end]}) + assert_receive {:warmup, pid}, 5_000 + ref = Process.monitor(pid) + assert_receive {:DOWN, ^ref, :process, ^pid, _} + refute_receive project_index_ready() + + Indexer.refresh(project) assert_receive project_index_ready(project: ^project) end - defp runtime_versions, do: %{erlang: "engine-erlang", elixir: "engine-elixir"} + test "failed warmup permits a later refresh", %{project: project} do + test_pid = self() + + patch(Search.Indexer, :warmup, fn _ -> + send(test_pid, {:warmup, self()}) + exit(:warmup_failed) + end) + + start_supervised!({Indexer, [project, create_index: fn _ -> :ok end]}) + assert_receive {:warmup, pid}, 5_000 + ref = Process.monitor(pid) + assert_receive {:DOWN, ^ref, :process, ^pid, _} + + Indexer.refresh(project) + assert_receive project_index_ready(project: ^project) + end + + test "skips warmup when the store has a persisted index", %{project: project} do + patch(Store, :load_status, :stale) + patch(Search.Indexer, :warmup, fn _ -> flunk("A persisted index does not need warmup") end) + start_supervised!({Indexer, project}) + assert_eventually(is_nil(:sys.get_state(Indexer.name(project)).task)) + end + + test "warmup stops when its owner is killed", %{project: project} do + test_pid = self() + + patch(Search.Indexer, :warmup, fn _ -> + send(test_pid, {:warmup, self()}) + Process.sleep(:infinity) + end) + + owner = start_supervised!({Indexer, project}) + assert_receive {:warmup, worker} + ref = Process.monitor(worker) + Process.exit(owner, :kill) + assert_receive {:DOWN, ^ref, :process, ^worker, :killed} + end + + test "coalesces refresh requests and reports readiness after the final refresh", %{ + project: project + } do + test_pid = self() + + start_supervised!( + {Indexer, + [ + project, + create_index: fn _ -> + send(test_pid, {:refresh, self()}) + + receive do + :finish -> :ok + end + end + ]} + ) + + Indexer.refresh(project) + assert_receive {:refresh, first}, 5_000 + Indexer.refresh(project) + Indexer.refresh(project) + :sys.get_state(Indexer.name(project)) + send(first, :finish) + + assert_receive {:refresh, second} + refute_receive project_index_ready() + send(second, :finish) + assert_receive project_index_ready(project: ^project) + refute_receive {:refresh, _} + end end diff --git a/apps/expert/test/expert/project/mix_project_test.exs b/apps/expert/test/expert/project/mix_project_test.exs new file mode 100644 index 000000000..6a3b85b74 --- /dev/null +++ b/apps/expert/test/expert/project/mix_project_test.exs @@ -0,0 +1,37 @@ +defmodule Expert.Project.MixProjectTest do + use ExUnit.Case + + alias Expert.Project.MixProject + alias Forge.Project + + @tag :tmp_dir + test "reads runtime versions for a bare project", %{tmp_dir: root} do + project = root |> Forge.Document.Path.to_uri() |> Project.bare() + assert {:ok, versions} = MixProject.runtime_versions(project) + assert versions == Forge.VM.Versions.current() + end + + @tag :tmp_dir + test "reads runtime versions even when mix.exs fails", %{tmp_dir: root} do + File.write!(Path.join(root, "mix.exs"), "raise \"mix.exs must not run\"") + project = root |> Forge.Document.Path.to_uri() |> Project.new() + assert {:ok, versions} = MixProject.runtime_versions(project) + assert versions == Forge.VM.Versions.current() + end + + @tag :tmp_dir + test "reads configuration after output from mix.exs", %{tmp_dir: root} do + File.write!(Path.join(root, "mix.exs"), """ + IO.puts("output from mix.exs") + defmodule ConfigurationTest.MixProject do + use Mix.Project + def project, do: [app: :example, version: "0.1.0", build_path: "custom_build"] + end + """) + + project = root |> Forge.Document.Path.to_uri() |> Project.new() + assert {:ok, config} = MixProject.configuration(project) + assert config.config[:app] == :example + assert config.build_path == Path.join(root, "custom_build/test") + end +end diff --git a/apps/expert/test/expert/project/node_test.exs b/apps/expert/test/expert/project/node_test.exs index 85aecb47a..53a75449f 100644 --- a/apps/expert/test/expert/project/node_test.exs +++ b/apps/expert/test/expert/project/node_test.exs @@ -7,8 +7,8 @@ defmodule Expert.Project.NodeTest do import Forge.Test.Fixtures alias Expert.EngineApi - alias Expert.Project.Indexer alias Expert.Project.Node, as: EngineNode + alias Expert.Project.SearchListener setup do project = project() @@ -95,7 +95,7 @@ defmodule Expert.Project.NodeTest do ) assert_receive {:registered, new_pid, [project_compiled() | _]}, :timer.seconds(15) - assert new_pid == Process.whereis(Indexer.name(project)) + assert new_pid == Process.whereis(SearchListener.name(project)) assert_receive {:compile, _force?}, :timer.seconds(15) refute_receive {:compile, _} end diff --git a/apps/expert/test/expert/project/search_listener_test.exs b/apps/expert/test/expert/project/search_listener_test.exs index f139ac25c..62f864b15 100644 --- a/apps/expert/test/expert/project/search_listener_test.exs +++ b/apps/expert/test/expert/project/search_listener_test.exs @@ -8,6 +8,8 @@ defmodule Expert.Project.SearchListenerTest do import Forge.Test.Fixtures alias Expert.EngineApi + alias Expert.Project.Indexer + alias Expert.Project.SearchListener alias Expert.Test.DispatchFake alias Forge.Project alias GenLSP.Notifications.WindowShowMessage @@ -15,10 +17,14 @@ defmodule Expert.Project.SearchListenerTest do setup do project = project() + test_pid = self() DispatchFake.start() - start_supervised!({Expert.Project.SearchListener, project}) + patch(Expert.Project.Node, :trigger_build, fn ^project, force? -> + send(test_pid, {:compile, force?}) + end) + start_supervised!({SearchListener, project}) {:ok, project: project} end @@ -40,4 +46,19 @@ defmodule Expert.Project.SearchListenerTest do }} end end + + test "compilation success and failure request index refreshes", %{project: project} do + test_pid = self() + patch(Indexer, :refresh, fn ^project -> send(test_pid, :refresh) end) + listener = SearchListener.name(project) + + send(listener, project_compiled(status: :success)) + assert_receive :refresh + send(listener, project_compiled(status: :error)) + assert_receive :refresh + end + + test "requests an incremental initial compile" do + assert_receive {:compile, false} + end end diff --git a/apps/expert/test/expert/project/warmup_test.exs b/apps/expert/test/expert/project/warmup_test.exs new file mode 100644 index 000000000..a48b75b9e --- /dev/null +++ b/apps/expert/test/expert/project/warmup_test.exs @@ -0,0 +1,62 @@ +defmodule Expert.Project.WarmupTest do + use ExUnit.Case, async: false + use Patch + + import Forge.Test.EventualAssertions + + alias Expert.EngineApi + alias Expert.EngineNode + alias Expert.Progress + alias Expert.Project.Indexer + alias Expert.Search.Store + alias Expert.Search.Store.Backends.Sqlite + alias Forge.Document + alias Forge.Project + + @tag :tmp_dir + test "source symbols become available while the Engine starts", %{tmp_dir: root} do + File.write!(Path.join(root, "example.ex"), "defmodule WarmupExample, do: def(run, do: :ok)") + project = root |> Document.Path.to_uri() |> Project.bare() + + start_supervised!({Sqlite, [project, runtime_versions: Forge.VM.Versions.current()]}) + start_supervised!({Store, [project, Sqlite]}) + start_supervised!({Expert.Search.Indexer.ModuleRegistry, project}) + start_supervised!({Task.Supervisor, name: Indexer.task_supervisor_name(project)}) + start_supervised!({Expert.Project.Store, []}) + patch(EngineApi, :call, fn _, _, _, _ -> flunk("Warmup must work without the Engine") end) + patch(Progress, :begin, fn _, _ -> {:error, :rejected} end) + + patch(Progress, :with_progress, fn _, fun -> + {:done, result, _message} = fun.(Progress.noop_token()) + result + end) + + test_pid = self() + + patch(EngineNode, :start, fn ^project, _token -> + send(test_pid, {:engine_start, self()}) + + receive do + :finish -> {:error, :test_stop} + end + end) + + start_supervised!({Indexer, project}) + + task = + Task.async(fn -> + Process.flag(:trap_exit, true) + Expert.Project.Node.start_link(project) + end) + + assert_receive {:engine_start, node_pid} + + assert_eventually( + match?({:ok, [_ | _]}, Store.exact(project, "WarmupExample.run/0", subtype: :definition)), + 5_000 + ) + + send(node_pid, :finish) + assert {:error, _} = Task.await(task) + end +end diff --git a/apps/expert/test/expert/provider/handlers/code_action_test.exs b/apps/expert/test/expert/provider/handlers/code_action_test.exs index 628b953b0..c07811c65 100644 --- a/apps/expert/test/expert/provider/handlers/code_action_test.exs +++ b/apps/expert/test/expert/provider/handlers/code_action_test.exs @@ -45,6 +45,7 @@ defmodule Expert.Provider.Handlers.CodeActionTest do EngineApi.schedule_compile(project, true) assert_receive project_compiled(), @project_ready_timeout + Indexer.refresh(project) assert_receive project_index_ready(project: ^project), @project_ready_timeout {:ok, project: project} diff --git a/apps/expert/test/expert/provider/handlers/find_references_test.exs b/apps/expert/test/expert/provider/handlers/find_references_test.exs index b8fc6aa6e..7c08eef99 100644 --- a/apps/expert/test/expert/provider/handlers/find_references_test.exs +++ b/apps/expert/test/expert/provider/handlers/find_references_test.exs @@ -63,12 +63,14 @@ defmodule Expert.Provider.Handlers.FindReferencesTest do describe "find references" do test "returns locations that the entity returns", %{project: project, uri: uri} do + assert Handlers.FindReferences.requires_engine?() == false project_uri = project.root_uri patch(References, :references, fn %{root_uri: ^project_uri}, %Analysis{document: document}, _position, - _ -> + _, + false -> locations = [ Location.new( Document.Range.new( @@ -97,7 +99,7 @@ defmodule Expert.Provider.Handlers.FindReferencesTest do end test "does not resolve a literal as its enclosing function call", %{project: project} do - patch(EngineApi, :resolve_entity, {:error, :unresolved}) + patch(EngineApi, :resolve_entity, fn _, _, _ -> flunk("called the Engine") end) path = file_path(project, Path.join("lib", "uses.ex")) {:ok, request} = build_request(path, 4, 25) @@ -120,7 +122,7 @@ defmodule Expert.Provider.Handlers.FindReferencesTest do assert {:ok, []} = handle(request, project) end - test "finds indexed references from a function declaration", %{ + test "finds references from a function declaration before the Engine is ready", %{ project: project, uri: uri } do @@ -201,7 +203,8 @@ defmodule Expert.Provider.Handlers.FindReferencesTest do {:error, :unresolved} end) - Expert.Project.Store.transition(project, :ready) + Expert.Project.Store.add_projects([project]) + assert Expert.Project.Store.transition(project, :ready) on_exit(fn -> Expert.Project.Store.transition(project, :pending) end) assert {:ok, []} = handle(request, project) diff --git a/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs b/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs index 0e31e20e7..ec40712be 100644 --- a/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs +++ b/apps/expert/test/expert/provider/handlers/go_to_definition_test.exs @@ -51,6 +51,7 @@ defmodule Expert.Provider.Handlers.GoToDefinitionTest do EngineApi.schedule_compile(project, true) assert_receive project_compiled(), @project_compile_timeout + Indexer.refresh(project) assert_receive project_index_ready(project: ^project), @project_index_timeout {:ok, project: project} @@ -92,10 +93,11 @@ defmodule Expert.Provider.Handlers.GoToDefinitionTest do describe "go to definition" do setup [:with_referenced_file] - test "resolves definitions from the manager index", %{ + test "uses the index before the Engine is ready", %{ project: project, uri: referenced_uri } do + assert Handlers.GoToDefinition.requires_engine?() == false patch(EngineApi, :definition, fn _, _, _ -> flunk("called the Engine") end) uses_file_path = file_path(project, Path.join("lib", "uses.ex")) @@ -105,7 +107,7 @@ defmodule Expert.Provider.Handlers.GoToDefinitionTest do assert Location.uri(location) == referenced_uri end - test "resolves local variables in the manager", %{project: project} do + test "finds a local variable before the Engine is ready", %{project: project} do patch(EngineApi, :definition, fn _, _, _ -> flunk("called the Engine") end) path = file_path(project, Path.join("lib", "my_definition.ex")) diff --git a/apps/expert/test/expert/search/indexer_test.exs b/apps/expert/test/expert/search/indexer_test.exs index cfb43c383..e180e0ee2 100644 --- a/apps/expert/test/expert/search/indexer_test.exs +++ b/apps/expert/test/expert/search/indexer_test.exs @@ -278,7 +278,7 @@ defmodule Expert.Search.IndexerTest do end) assert :ok = - Indexer.create_index(project, paths: paths) + Indexer.warmup(project, paths: paths) assert_receive {:registry_application, :registry_dependency} @@ -309,7 +309,7 @@ defmodule Expert.Search.IndexerTest do } assert :ok = - Indexer.create_index(project, paths: paths) + Indexer.warmup(project, paths: paths) function_subject = Forge.Formats.mfa(module, :public_fun, 0) @@ -736,6 +736,38 @@ defmodule Expert.Search.IndexerTest do end) end + @tag :tmp_dir + test "retains source entries when compilation creates a project beam", %{tmp_dir: tmp_dir} do + %{beam_path: beam_path, dep_file: source_path, module: module, project: project} = + with_beam_dependency(tmp_dir, rewrite_source?: false) + + start_registry(project) + + assert :ok = + Indexer.warmup(project, + paths: %Paths{source_paths: [source_path], beam_paths: []} + ) + + patch(Paths, :for_project, fn ^project -> + %Paths{ + source_paths: [source_path], + beam_paths: [beam_path], + applications: %{Path.dirname(beam_path) => :beam_dep} + } + end) + + private_subject = Forge.Formats.mfa(module, :private_fun, 0) + update_index(project) + + assert Enum.any?(FakeBackend.entries(), fn entry -> + entry.subject == private_subject and entry.subtype == :definition + end) + + assert Enum.any?(FakeBackend.entries(), fn entry -> + entry.subject == private_subject and entry.subtype == :reference + end) + end + @tag :tmp_dir test "clears entries when a dependency is removed but its beam remains", %{tmp_dir: tmp_dir} do %{app_root: app_root, beam_path: beam_path, dep_file: dep_file, project: project} =