[WIP] implement recursive analysis infrastructure

src/analysis/Interpreter.jl

@@ -6,7 +6,7 @@ using JuliaSyntax: JuliaSyntax as JS
 using JET: CC, JET
 using ..JETLS:
     AnalysisEntry, FullAnalysisInfo, SavedFileInfo, Server,
-    JETLS_DEV_MODE, build_tree!, get_saved_file_info, yield_to_endpoint, send
+    JETLS_DEV_MODE, build_tree!, get_saved_file_info, yield_to_endpoint, send, get_config
 using ..JETLS.URIs2
 using ..JETLS.LSP
 using ..JETLS.Analyzer
@@ -18,17 +18,18 @@ Counter() = Counter(0)
 increment!(counter::Counter) = counter.count += 1
 Base.getindex(counter::Counter) = counter.count
 
-struct LSInterpreter{S<:Server, I<:FullAnalysisInfo} <: JET.ConcreteInterpreter
-    server::S
-    info::I
-    analyzer::LSAnalyzer
-    counter::Counter
-    state::JET.InterpretationState
+mutable struct LSInterpreter{S<:Server, I<:FullAnalysisInfo} <: JET.ConcreteInterpreter
+    const server::S
+    const info::I
+    const analyzer::LSAnalyzer
+    const counter::Counter
+    const state::JET.InterpretationState
+    current_pkgid::Union{Base.PkgId, Nothing}
     function LSInterpreter(server::S, info::I, analyzer::LSAnalyzer, counter::Counter) where S<:Server where I<:FullAnalysisInfo
         return new{S,I}(server, info, analyzer, counter)
     end
     function LSInterpreter(server::S, info::I, analyzer::LSAnalyzer, counter::Counter, state::JET.InterpretationState) where S<:Server where I<:FullAnalysisInfo
-        return new{S,I}(server, info, analyzer, counter, state)
+        return new{S,I}(server, info, analyzer, counter, state,  state.config.pkgid)
     end
 end
 
@@ -145,4 +146,199 @@ function JET.try_read_file(interp::LSInterpreter, include_context::Module, filen
     return read(filename, String)
 end
 
+
+function should_recursive_analyze(interp::LSInterpreter, dep::Symbol)
+    if interp.info.reanalyze && !get_config(server.state.config_manager, "recursive_analysis", "reanalyze")
+        return false
+    end
+    server = interp.server
+    server_state = server.state
+    interp_state = JET.InterpretationState(interp)
+    depth = interp_state.pkg_mod_depth
+
+    max_depth = get_config(server_state.config_manager, "recursive_analysis", "max_depth")
+    max_depth < depth && return false
+
+    exclude = get_config(server_state.config_manager, "recursive_analysis", "exclude")
+    depstr = String(dep)
+    # XXX: This is temporary implementation.
+    #      In the future, this method support `Regex` or `Glob` like object.
+    depstr in exclude && return false
+
+    return true
+end
+
+"""
+    maybe_introduce_mod(required_pkgid::Base.PkgId, caller_mod::Module, dep::Symbol)
+
+If the module is already loaded (i.e. `dep` in `Base.loaded_modules`),
+introduce it to `caller_mod` as `const dep = maybe_mod`.
+
+If the module is not loaded or failed to introduce, return `false`.
+(If failed to introduce, error reports will be added to `state`.)
+
+If succeeded, return `true`.
+"""
+function maybe_introduce_mod!(required_pkgid::Base.PkgId, caller_mod::Module, dep::Symbol, state::JET.InterpretationState)
+    maybe_mod = Base.maybe_root_module(required_pkgid)
+    maybe_mod === nothing && return false
+    @assert maybe_mod isa Module
+    res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do
+        Core.eval(caller_mod, :(const $dep = $maybe_mod))
+    end
+    return res !== nothing
+end
+
+
+"""
+    include_on___toplevel__(required_pkgid::Base.PkgId, required_env::AbstractString, interp::LSInterpreter) -> Union{Module, Nothing}
+
+Load the package identified by `required_pkgid` in the environment `required_env` following process:
+1. Locate the package path using `Base.locate_package`.
+  - If not found, return `nothing`. (This typically happens when the `instantiate` step is not done.)
+2. Load the package into `Base.__toplevel__` using `Base.include`.
+3. Return the loaded module using `Base.maybe_root_module`.
+  - If failed, return `nothing`.
+"""
+function include_on___toplevel__(required_pkgid::Base.PkgId, required_env::AbstractString, interp::LSInterpreter)
+    path = Base.locate_package(required_pkgid, required_env)
+    if path === nothing
+        return nothing
+    end
+    required_uuid = required_pkgid.uuid
+    required_uuid = (required_uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, required_uuid))
+    old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), Base.__toplevel__)
+    if required_uuid !== old_uuid
+        ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, required_uuid)
+    end
+    old_pkgid = interp.current_pkgid
+    interp.current_pkgid = required_pkgid
+    try
+        JET.handle_include(
+            interp,
+            Base.include,
+            [Base.__toplevel__, path])
+    finally
+        interp.current_pkgid = old_pkgid
+        loaded_mod = Base.maybe_root_module(required_pkgid)
+        if required_uuid !== old_uuid
+            ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, old_uuid)
+        end
+        if loaded_mod === nothing
+            @warn "Failed to load module $required_pkgid from $path"
+            return nothing
+        end
+        return loaded_mod
+    end
+end
+
+function JET.usemodule_with_err_handling(interp::LSInterpreter, ex::Expr)
+    # In 1.13, lowerd form of `using`/`import` is changed
+    # TODO: support 1.13 and later
+    @static if VERSION >= v"1.13.0"
+        return @invoke JET.usemodule_with_err_handling(interp::JET.ConcreteInterpreter, ex::Expr)
+    end
+
+    state = JET.InterpretationState(interp)
+    caller_mod = state.context
+    Meta.isexpr(ex, (:export, :public)) && @goto eval_usemodule
+    current_pkgid = interp.current_pkgid
+    if current_pkgid !== nothing
+        module_usage = JET.pattern_match_module_usage(ex)
+        (; modpath) = module_usage
+        dep = first(modpath)::Symbol
+        if !(dep === :. || # relative module doesn't need to be fixed
+             dep === :Base || dep === :Core) # modules available by default
+            if dep === Symbol(current_pkgid.name)
+                # it's somehow allowed to use the package itself without the relative module path,
+                # so we need to special case it and fix it to use the relative module path
+                for _ = 1:state.pkg_mod_depth
+                    pushfirst!(modpath, :.)
+                end
+            else
+                dependencies = interp.server.state.dependencies
+                depstr = String(dep)
+                required_pkgenv = Base.identify_package_env(caller_mod, depstr)
+                if required_pkgenv === nothing
+                    local report = JET.DependencyError(current_pkgid.name, depstr, state.filename, state.curline)
+                    JET.add_toplevel_error_report!(state, report)
+                    return nothing
+                end
+                required_pkgid, required_env = required_pkgenv
+                if required_pkgid ∉ dependencies
+                    if !should_recursive_analyze(interp, dep)
+                        # TODO: refactor JET to avoid code duplication
+                        res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do
+                            Core.eval(caller_mod, :(const $dep = Base.require($required_pkgid)))
+                        end
+                        if res === nothing
+                            @warn "Failed to set module to $dep in $caller_mod"
+                            return nothing
+                        end
+                    else
+                        maybe_mod = include_on___toplevel__(required_pkgid, required_env, interp)
+                        if maybe_mod === nothing
+                            local report = JET.DependencyError(current_pkgid.name, depstr, state.filename, state.curline)
+                            JET.add_toplevel_error_report!(state, report)
+                            return nothing
+                        end
+
+                        res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do
+                            Core.eval(caller_mod, :(const $dep = $maybe_mod))
+                        end
+                        if res === nothing
+                            @warn "Failed to set module to $dep in $caller_mod"
+                            return nothing
+                        end
+                    end
+                    push!(interp.server.state.dependencies, required_pkgid)
+                else
+                    # we already see this package, so introduce it.
+                    introduced = maybe_introduce_mod!(required_pkgid, caller_mod, dep, state)
+                    if !introduced
+                        @warn "Failed to introduce module $dep in $caller_mod in spite of being already seen"
+                        return nothing
+                    end
+                end
+                pushfirst!(modpath, :.)
+            end
+            fixed_module_usage = JET.ModuleUsage(module_usage; modpath)
+            ex = JET.form_module_usage(fixed_module_usage)
+        elseif dep === :.
+            # The syntax `import ..Submod` refers to the name that is available within
+            # a parent module specified by the number of `.` dots, indicating how many
+            # levels up the module hierarchy to go. However, when it comes to package
+            # loading, it seems to work regardless of the number of dots. For now, in
+            # `report_package`, adjust `modpath` here to mimic the package loading behavior.
+            topmodidx = findfirst(@nospecialize(mp)->mp!==:., modpath)::Int
+            topmodsym = modpath[topmodidx]
+            curmod = caller_mod
+            for i = 1:(topmodidx-1)
+                if topmodsym isa Symbol && isdefined(curmod, topmodsym)
+                    modpath = modpath[topmodidx:end]
+                    for j = 1:i
+                        pushfirst!(modpath, :.)
+                    end
+                    fixed_module_usage = JET.ModuleUsage(module_usage; modpath)
+                    ex = JET.form_module_usage(fixed_module_usage)
+                    break
+                else
+                    curmod = parentmodule(curmod)
+                end
+            end
+        end
+    end
+
+    @label eval_usemodule
+
+    # # `scrub_offset = 1`: `Core.eval`
+    # Executing `using/import` is somewhat redundant, since the module is already loaded
+    # and the remaining work is only to introduce names, but this doesn't affect results.
+    # TODO: Refactor using `Core._import` or similar.
+    JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do
+        Core.eval(caller_mod, ex)
+        true
+    end
+
+    end
 end # module Interpreter

src/analysis/full-analysis.jl

@@ -266,6 +266,17 @@ function initiate_analysis_unit!(server::Server, uri::URI; token::Union{Nothing,
     return analysis_unit
 end
 
+function clear_dependencies_cache!(server)
+    dependencies = server.state.dependencies
+    for dep in dependencies
+        # TODO: make configurable cache clearing condition
+        if !Base.is_stdlib(dep)
+            Base.unreference_module(dep)
+        end
+    end
+    empty!(dependencies)
+end
+
 function reanalyze!(server::Server, analysis_unit::AnalysisUnit; token::Union{Nothing,ProgressToken}=nothing)
     state = server.state
     analysis_result = analysis_unit.result
@@ -288,6 +299,10 @@ function reanalyze!(server::Server, analysis_unit::AnalysisUnit; token::Union{No
     entry = analysis_unit.entry
     n_files = length(values(analysis_unit.result.successfully_analyzed_file_infos))
 
+    if get_config(server.state.config_manager, "recursive_analysis", "reanalyze")
+        clear_dependencies_cache!(server)
+    end
+
     # manually dispatch here for the maximum inferrability
     if entry isa ScriptAnalysisEntry
         info = FullAnalysisInfo(entry, token, #=reanalyze=#true, n_files)

src/types.jl

@@ -237,6 +237,14 @@ global DEFAULT_CONFIG::Dict{String,Any} = Dict{String,Any}(
     "testrunner" => Dict{String,Any}(
         "executable" => "testrunner"
     ),
+    "recursive_analysis" => Dict{String, Any}(
+        "max_depth" => 0,
+        # In the future, this should use `Regex` or `Glob` like patterns.
+        # Currently, since schema-related mechanisms are not fully developed,
+        # only plain string and exact match is supported.
+        "exclude" => String[],
+        "reanalyze" => true
+    )
 )
 
 global CONFIG_RELOAD_REQUIRED::Dict{String,Any} = Dict{String,Any}(
@@ -249,6 +257,11 @@ global CONFIG_RELOAD_REQUIRED::Dict{String,Any} = Dict{String,Any}(
     "testrunner" => Dict{String,Any}(
         "executable" => false
     ),
+    "recursive_analysis" => Dict{String, Any}(
+        "max_depth" => true,
+        "exclude" => true,
+        "reanalyze" => true
+    )
 )
 
 WatchedConfigFiles() = WatchedConfigFiles(String["__DEFAULT_CONFIG__"], Dict{String,Any}[DEFAULT_CONFIG])
@@ -314,6 +327,7 @@ mutable struct ServerState
     const currently_requested::Dict{String,RequestCaller}
     const currently_registered::Set{Registered}
     const config_manager::ConfigManager
+    const dependencies::Set{Base.PkgId}
     encoding::PositionEncodingKind.Ty
     root_path::String
     root_env_path::String
@@ -329,6 +343,7 @@ mutable struct ServerState
             #=currently_requested=# Dict{String,RequestCaller}(),
             #=currently_registered=# Set{Registered}(),
             #=config_manager=# ConfigManager(),
+            #=dependencies=# Set{Base.PkgId}(),
             #=encoding=# PositionEncodingKind.UTF16, # initialize with UTF16 (for tests)
         )
     end