From 92bdb985554bcc7a09e8d359ce54f529293c4761 Mon Sep 17 00:00:00 2001 From: abap34 Date: Fri, 22 Aug 2025 21:27:11 +0900 Subject: [PATCH 1/3] implement recursive analysis infrastructure --- src/analysis/Interpreter.jl | 177 ++++++++++++++++++++++++++++++++-- src/analysis/full-analysis.jl | 13 +++ src/types.jl | 13 +++ 3 files changed, 195 insertions(+), 8 deletions(-) diff --git a/src/analysis/Interpreter.jl b/src/analysis/Interpreter.jl index bcc0543bf..45d3789a8 100644 --- a/src/analysis/Interpreter.jl +++ b/src/analysis/Interpreter.jl @@ -6,7 +6,7 @@ using JuliaSyntax: JuliaSyntax as JS using JET: CC, JET using ..JETLS: AnalysisEntry, FullAnalysisInfo, SavedFileInfo, Server, - JETLS_DEV_MODE, build_tree!, get_saved_file_info, yield_to_endpoint, send + JETLS_DEV_MODE, build_tree!, get_saved_file_info, yield_to_endpoint, send, get_config using ..JETLS.URIs2 using ..JETLS.LSP using ..JETLS.Analyzer @@ -18,17 +18,18 @@ Counter() = Counter(0) increment!(counter::Counter) = counter.count += 1 Base.getindex(counter::Counter) = counter.count -struct LSInterpreter{S<:Server, I<:FullAnalysisInfo} <: JET.ConcreteInterpreter - server::S - info::I - analyzer::LSAnalyzer - counter::Counter - state::JET.InterpretationState +mutable struct LSInterpreter{S<:Server, I<:FullAnalysisInfo} <: JET.ConcreteInterpreter + const server::S + const info::I + const analyzer::LSAnalyzer + const counter::Counter + const state::JET.InterpretationState + current_pkgid::Union{Base.PkgId, Nothing} function LSInterpreter(server::S, info::I, analyzer::LSAnalyzer, counter::Counter) where S<:Server where I<:FullAnalysisInfo return new{S,I}(server, info, analyzer, counter) end function LSInterpreter(server::S, info::I, analyzer::LSAnalyzer, counter::Counter, state::JET.InterpretationState) where S<:Server where I<:FullAnalysisInfo - return new{S,I}(server, info, analyzer, counter, state) + return new{S,I}(server, info, analyzer, counter, state, state.config.pkgid) end end @@ -145,4 +146,164 @@ function JET.try_read_file(interp::LSInterpreter, include_context::Module, filen return read(filename, String) end + +function should_recursive_analyze(interp::LSInterpreter, dep::Symbol) + server = interp.server + server_state = server.state + interp_state = JET.InterpretationState(interp) + depth = interp_state.pkg_mod_depth + + max_depth = get_config(server_state.config_manager, "recursive_analysis", "max_depth") + max_depth < depth && return false + + exclude = get_config(server_state.config_manager, "recursive_analysis", "exclude") + depstr = String(dep) + # XXX: This is temporary implementation. + # In the future, this method support `Regex` or `Glob` like object. + depstr in exclude && return false + + return true +end + + +function JET.usemodule_with_err_handling(interp::LSInterpreter, ex::Expr) + # In 1.13, lowerd form of `using`/`import` is changed + # TODO: support 1.13 and later + @static if VERSION >= v"1.13.0" + return @invoke JET.usemodule_with_err_handling(interp::JET.ConcreteInterpreter, ex::Expr) + end + + state = JET.InterpretationState(interp) + caller_mod = state.context + Meta.isexpr(ex, (:export, :public)) && @goto eval_usemodule + current_pkgid = interp.current_pkgid + if current_pkgid !== nothing + module_usage = JET.pattern_match_module_usage(ex) + (; modpath) = module_usage + dep = first(modpath)::Symbol + if !(dep === :. || # relative module doesn't need to be fixed + dep === :Base || dep === :Core) # modules available by default + if dep === Symbol(current_pkgid.name) + # it's somehow allowed to use the package itself without the relative module path, + # so we need to special case it and fix it to use the relative module path + for _ = 1:state.pkg_mod_depth + pushfirst!(modpath, :.) + end + else + dependencies = interp.server.state.dependencies + depstr = String(dep) + required_pkgenv = Base.identify_package_env(caller_mod, depstr) + if required_pkgenv === nothing + local report = JET.DependencyError(current_pkgid.name, depstr, state.filename, state.curline) + JET.add_toplevel_error_report!(state, report) + return nothing + end + required_pkgid, required_env = required_pkgenv + if required_pkgid ∉ dependencies + maybe_mod = Base.maybe_root_module(required_pkgid) + if maybe_mod isa Module + @assert nameof(maybe_mod) == dep + res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do + Core.eval(caller_mod, :(const $dep = $maybe_mod)) + true + end + if res === nothing + @warn "Failed to set module to $dep in $mod" + return nothing + end + else + if !should_recursive_analyze(interp, dep) + res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do + Core.eval(caller_mod, :(const $dep = Base.require($required_pkgid))) + true + end + if res === nothing + @warn "Failed to set module to $dep in $mod" + return nothing + end + else + path = Base.locate_package(required_pkgid, required_env) + # without `instantiate`, sometimes required_pkgenv is not `nothing` but path is nothing + if path === nothing + local report = JET.DependencyError(current_pkgid.name, depstr, state.filename, state.curline) + JET.add_toplevel_error_report!(state, report) + return nothing + end + required_uuid = required_pkgid.uuid + required_uuid = (required_uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, required_uuid)) + old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), Base.__toplevel__) + if required_uuid !== old_uuid + ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, required_uuid) + end + old_pkgid = interp.current_pkgid + interp.current_pkgid = required_pkgid + try + JET.handle_include( + interp, + Base.include, + [Base.__toplevel__, path]) + finally + interp.current_pkgid = old_pkgid + loaded_mod = Base.maybe_root_module(required_pkgid) + if loaded_mod === nothing + @warn "Failed to get $dep" + return nothing + end + res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do + Core.eval(caller_mod, :(const $dep = $loaded_mod)) + true + end + if res === nothing + @warn "Failed to set module to $dep in $mod" + return nothing + end + if required_uuid !== old_uuid + ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, old_uuid) + end + end + end + end + push!(interp.server.state.dependencies, required_pkgid) + end + pushfirst!(modpath, :.) + end + fixed_module_usage = JET.ModuleUsage(module_usage; modpath) + ex = JET.form_module_usage(fixed_module_usage) + elseif dep === :. + # The syntax `import ..Submod` refers to the name that is available within + # a parent module specified by the number of `.` dots, indicating how many + # levels up the module hierarchy to go. However, when it comes to package + # loading, it seems to work regardless of the number of dots. For now, in + # `report_package`, adjust `modpath` here to mimic the package loading behavior. + topmodidx = findfirst(@nospecialize(mp)->mp!==:., modpath)::Int + topmodsym = modpath[topmodidx] + curmod = caller_mod + for i = 1:(topmodidx-1) + if topmodsym isa Symbol && isdefined(curmod, topmodsym) + modpath = modpath[topmodidx:end] + for j = 1:i + pushfirst!(modpath, :.) + end + fixed_module_usage = JET.ModuleUsage(module_usage; modpath) + ex = JET.form_module_usage(fixed_module_usage) + break + else + curmod = parentmodule(curmod) + end + end + end + end + + @label eval_usemodule + + # # `scrub_offset = 1`: `Core.eval` + # Executing `using/import` is somewhat redundant, since the module is already loaded + # and the remaining work is only to introduce names, but this doesn't affect results. + # TODO: Refactor using `Core._import` or similar. + JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do + Core.eval(caller_mod, ex) + true + end + + end end # module Interpreter diff --git a/src/analysis/full-analysis.jl b/src/analysis/full-analysis.jl index 16abe55dc..0a70ffdc4 100644 --- a/src/analysis/full-analysis.jl +++ b/src/analysis/full-analysis.jl @@ -266,6 +266,17 @@ function initiate_analysis_unit!(server::Server, uri::URI; token::Union{Nothing, return analysis_unit end +function clear_dependencies_cache!(server) + dependencies = server.state.dependencies + for dep in dependencies + # TODO: make configurable cache clearing condition + if !Base.is_stdlib(dep) + Base.unreference_module(dep) + end + end + empty!(dependencies) +end + function reanalyze!(server::Server, analysis_unit::AnalysisUnit; token::Union{Nothing,ProgressToken}=nothing) state = server.state analysis_result = analysis_unit.result @@ -288,6 +299,8 @@ function reanalyze!(server::Server, analysis_unit::AnalysisUnit; token::Union{No entry = analysis_unit.entry n_files = length(values(analysis_unit.result.successfully_analyzed_file_infos)) + clear_dependencies_cache!(server) + # manually dispatch here for the maximum inferrability if entry isa ScriptAnalysisEntry info = FullAnalysisInfo(entry, token, #=reanalyze=#true, n_files) diff --git a/src/types.jl b/src/types.jl index 44c75d4a5..2a1c15caf 100644 --- a/src/types.jl +++ b/src/types.jl @@ -237,6 +237,13 @@ global DEFAULT_CONFIG::Dict{String,Any} = Dict{String,Any}( "testrunner" => Dict{String,Any}( "executable" => "testrunner" ), + "recursive_analysis" => Dict{String, Any}( + "max_depth" => 0, + # In the future, this should use `Regex` or `Glob` like patterns. + # Currently, since schema-related mechanisms are not fully developed, + # only plain string and exact match is supported. + "exclude" => String[] + ) ) global CONFIG_RELOAD_REQUIRED::Dict{String,Any} = Dict{String,Any}( @@ -249,6 +256,10 @@ global CONFIG_RELOAD_REQUIRED::Dict{String,Any} = Dict{String,Any}( "testrunner" => Dict{String,Any}( "executable" => false ), + "recursive_analysis" => Dict{String, Any}( + "max_depth" => true, + "exclude" => true + ) ) WatchedConfigFiles() = WatchedConfigFiles(String["__DEFAULT_CONFIG__"], Dict{String,Any}[DEFAULT_CONFIG]) @@ -314,6 +325,7 @@ mutable struct ServerState const currently_requested::Dict{String,RequestCaller} const currently_registered::Set{Registered} const config_manager::ConfigManager + const dependencies::Set{Base.PkgId} encoding::PositionEncodingKind.Ty root_path::String root_env_path::String @@ -329,6 +341,7 @@ mutable struct ServerState #=currently_requested=# Dict{String,RequestCaller}(), #=currently_registered=# Set{Registered}(), #=config_manager=# ConfigManager(), + #=dependencies=# Set{Base.PkgId}(), #=encoding=# PositionEncodingKind.UTF16, # initialize with UTF16 (for tests) ) end From 7ebdd9df1c31c64aa86cebc7267d1a73632eacc6 Mon Sep 17 00:00:00 2001 From: abap34 Date: Sat, 6 Sep 2025 17:13:43 +0900 Subject: [PATCH 2/3] refactor --- src/analysis/Interpreter.jl | 142 ++++++++++++++++++++++-------------- 1 file changed, 87 insertions(+), 55 deletions(-) diff --git a/src/analysis/Interpreter.jl b/src/analysis/Interpreter.jl index 45d3789a8..06724a0c3 100644 --- a/src/analysis/Interpreter.jl +++ b/src/analysis/Interpreter.jl @@ -165,6 +165,69 @@ function should_recursive_analyze(interp::LSInterpreter, dep::Symbol) return true end +""" + maybe_introduce_mod(required_pkgid::Base.PkgId, caller_mod::Module, dep::Symbol) + +If the module is already loaded (i.e. `dep` in `Base.loaded_modules`), +introduce it to `caller_mod` as `const dep = maybe_mod`. + +If the module is not loaded or failed to introduce, return `false`. +(If failed to introduce, error reports will be added to `state`.) + +If succeeded, return `true`. +""" +function maybe_introduce_mod!(required_pkgid::Base.PkgId, caller_mod::Module, dep::Symbol, state::JET.InterpretationState) + maybe_mod = Base.maybe_root_module(required_pkgid) + maybe_mod === nothing && return false + @assert maybe_mod isa Module + res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do + Core.eval(caller_mod, :(const $dep = $maybe_mod)) + end + return res !== nothing +end + + +""" + include_on___toplevel__(required_pkgid::Base.PkgId, required_env::AbstractString, interp::LSInterpreter) -> Union{Module, Nothing} + +Load the package identified by `required_pkgid` in the environment `required_env` following process: +1. Locate the package path using `Base.locate_package`. + - If not found, return `nothing`. (This typically happens when the `instantiate` step is not done.) +2. Load the package into `Base.__toplevel__` using `Base.include`. +3. Return the loaded module using `Base.maybe_root_module`. + - If failed, return `nothing`. +""" +function include_on___toplevel__(required_pkgid::Base.PkgId, required_env::AbstractString, interp::LSInterpreter) + path = Base.locate_package(required_pkgid, required_env) + if path === nothing + return nothing + end + required_uuid = required_pkgid.uuid + required_uuid = (required_uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, required_uuid)) + old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), Base.__toplevel__) + if required_uuid !== old_uuid + ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, required_uuid) + end + old_pkgid = interp.current_pkgid + interp.current_pkgid = required_pkgid + try + JET.handle_include( + interp, + Base.include, + [Base.__toplevel__, path]) + finally + interp.current_pkgid = old_pkgid + loaded_mod = Base.maybe_root_module(required_pkgid) + if required_uuid !== old_uuid + ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, old_uuid) + end + if loaded_mod === nothing + @warn "Failed to load module $required_pkgid from $path" + return nothing + end + return loaded_mod + end +end function JET.usemodule_with_err_handling(interp::LSInterpreter, ex::Expr) # In 1.13, lowerd form of `using`/`import` is changed @@ -200,70 +263,39 @@ function JET.usemodule_with_err_handling(interp::LSInterpreter, ex::Expr) end required_pkgid, required_env = required_pkgenv if required_pkgid ∉ dependencies - maybe_mod = Base.maybe_root_module(required_pkgid) - if maybe_mod isa Module - @assert nameof(maybe_mod) == dep + if !should_recursive_analyze(interp, dep) + # TODO: refactor JET to avoid code duplication res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do - Core.eval(caller_mod, :(const $dep = $maybe_mod)) - true + Core.eval(caller_mod, :(const $dep = Base.require($required_pkgid))) end if res === nothing - @warn "Failed to set module to $dep in $mod" + @warn "Failed to set module to $dep in $caller_mod" return nothing end else - if !should_recursive_analyze(interp, dep) - res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do - Core.eval(caller_mod, :(const $dep = Base.require($required_pkgid))) - true - end - if res === nothing - @warn "Failed to set module to $dep in $mod" - return nothing - end - else - path = Base.locate_package(required_pkgid, required_env) - # without `instantiate`, sometimes required_pkgenv is not `nothing` but path is nothing - if path === nothing - local report = JET.DependencyError(current_pkgid.name, depstr, state.filename, state.curline) - JET.add_toplevel_error_report!(state, report) - return nothing - end - required_uuid = required_pkgid.uuid - required_uuid = (required_uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, required_uuid)) - old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), Base.__toplevel__) - if required_uuid !== old_uuid - ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, required_uuid) - end - old_pkgid = interp.current_pkgid - interp.current_pkgid = required_pkgid - try - JET.handle_include( - interp, - Base.include, - [Base.__toplevel__, path]) - finally - interp.current_pkgid = old_pkgid - loaded_mod = Base.maybe_root_module(required_pkgid) - if loaded_mod === nothing - @warn "Failed to get $dep" - return nothing - end - res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do - Core.eval(caller_mod, :(const $dep = $loaded_mod)) - true - end - if res === nothing - @warn "Failed to set module to $dep in $mod" - return nothing - end - if required_uuid !== old_uuid - ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, old_uuid) - end - end + maybe_mod = include_on___toplevel__(required_pkgid, required_env, interp) + if maybe_mod === nothing + local report = JET.DependencyError(current_pkgid.name, depstr, state.filename, state.curline) + JET.add_toplevel_error_report!(state, report) + return nothing + end + + res = JET.with_err_handling(JET.general_err_handler, state; scrub_offset=1) do + Core.eval(caller_mod, :(const $dep = $maybe_mod)) + end + if res === nothing + @warn "Failed to set module to $dep in $caller_mod" + return nothing end end push!(interp.server.state.dependencies, required_pkgid) + else + # we already see this package, so introduce it. + introduced = maybe_introduce_mod!(required_pkgid, caller_mod, dep, state) + if !introduced + @warn "Failed to introduce module $dep in $caller_mod in spite of being already seen" + return nothing + end end pushfirst!(modpath, :.) end From e644c42cbd7ec3a54b7095a2cfb45dd1f8c16dc7 Mon Sep 17 00:00:00 2001 From: abap34 Date: Sat, 6 Sep 2025 23:34:12 +0900 Subject: [PATCH 3/3] make dependency package reanalysis configurable --- src/analysis/Interpreter.jl | 3 +++ src/analysis/full-analysis.jl | 4 +++- src/types.jl | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/analysis/Interpreter.jl b/src/analysis/Interpreter.jl index 06724a0c3..38b606db4 100644 --- a/src/analysis/Interpreter.jl +++ b/src/analysis/Interpreter.jl @@ -148,6 +148,9 @@ end function should_recursive_analyze(interp::LSInterpreter, dep::Symbol) + if interp.info.reanalyze && !get_config(server.state.config_manager, "recursive_analysis", "reanalyze") + return false + end server = interp.server server_state = server.state interp_state = JET.InterpretationState(interp) diff --git a/src/analysis/full-analysis.jl b/src/analysis/full-analysis.jl index 0a70ffdc4..8a415fa82 100644 --- a/src/analysis/full-analysis.jl +++ b/src/analysis/full-analysis.jl @@ -299,7 +299,9 @@ function reanalyze!(server::Server, analysis_unit::AnalysisUnit; token::Union{No entry = analysis_unit.entry n_files = length(values(analysis_unit.result.successfully_analyzed_file_infos)) - clear_dependencies_cache!(server) + if get_config(server.state.config_manager, "recursive_analysis", "reanalyze") + clear_dependencies_cache!(server) + end # manually dispatch here for the maximum inferrability if entry isa ScriptAnalysisEntry diff --git a/src/types.jl b/src/types.jl index 2a1c15caf..985bcf572 100644 --- a/src/types.jl +++ b/src/types.jl @@ -242,7 +242,8 @@ global DEFAULT_CONFIG::Dict{String,Any} = Dict{String,Any}( # In the future, this should use `Regex` or `Glob` like patterns. # Currently, since schema-related mechanisms are not fully developed, # only plain string and exact match is supported. - "exclude" => String[] + "exclude" => String[], + "reanalyze" => true ) ) @@ -258,7 +259,8 @@ global CONFIG_RELOAD_REQUIRED::Dict{String,Any} = Dict{String,Any}( ), "recursive_analysis" => Dict{String, Any}( "max_depth" => true, - "exclude" => true + "exclude" => true, + "reanalyze" => true ) )