From 07227f631b6b9b2c8538cee47c731f6d9c7e1f24 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 15 Jun 2026 20:31:15 +0000 Subject: [PATCH 1/6] compiler/codegen: fix missing edge and backedge updates on CI equivalence swaps When a CodeInstance `callee` is replaced by an equivalent `cached` CI (via `jl_get_ci_equiv` in `add_codeinsts_to_jit!` and `compile!`) or when a new CI is created as a copy (via `copy_to_mi_cache`), the edges/backedges were not updated. As a result, any caller whose edges svec contained `callee` by identity would not be invalidated when `cached` was invalidated, since `callee` was not registered in any MethodInstance's backedges. Fix by introducing `link_ci_equiv!` on the Julia side, which: 1. Appends `cached` to `callee.edges` so that `_invalidate_backedges`'s pointer-identity check (`edge == replaced_ci`) matches when `cached` is being invalidated. 2. Registers `callee` in `cached`'s MI's backedges (via `store_backedges`) so the invalidation walk can reach `callee` when `cached` is invalidated. 3. Promotes `callee` to the current world via `jl_promote_ci_to_current` so it participates in the normal validity regime. For `copy_to_mi_cache` on the C side, resolve the long-standing TODO by adding `store_backedges_c` (a C mirror of the Julia `store_backedges` function that parses the forward-edges svec format) and calling `jl_promote_ci_to_current` on the newly created CI. Also add `jl_codeinst_set_edges` as an exported C helper to atomically update a CodeInstance's edges field with the proper write barrier. Co-Authored-By: Claude Sonnet 4.6 --- Compiler/src/typeinfer.jl | 35 +++++++++++++++++++++-- src/gf.c | 60 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index fa5102e8b7b04..8d3eed5d84d8a 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -818,6 +818,31 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) nothing end +# When `callee` (not in the global MI cache) is being replaced by an equivalent `cached` +# (in the MI cache) for JIT compilation, ensure callee will be properly invalidated if +# cached is invalidated later. We do this by making callee depend on cached (adding cached +# to callee.edges) and registering callee in cached's MI backedges. +function link_ci_equiv!(callee::CodeInstance, cached::CodeInstance) + callee === cached && return true + mi = get_ci_mi(cached) + isa(mi.def, Method) || return true + # check if callee already has cached in its edges + old_edges = callee.edges::SimpleVector + for e in old_edges + e === cached && return true + end + validation_world = get_world_counter() + if callee.max_world > validation_world + # append cached to callee's edges so _invalidate_backedges can find it by identity + new_edges = Core.svec(old_edges..., cached) + @atomic :release callee.edges = new_edges + # register callee in cached's MI backedges so invalidation of cached propagates to callee + store_backedges(callee, Core.svec(cached)) + end + # after storing new backedges, use method if it is still legal + return cached.max_world > callee.max_world +end + function compute_edges!(sv::InferenceState) edges = sv.edges for i in 1:length(sv.stmt_info) @@ -1746,8 +1771,10 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn code_cache(workqueue.interp)[mi] = callee else # use an existing CI from the cache, if there is available one that is compatible - callee === ci && (ci = cached) - callee = cached + if link_ci_equiv!(callee, cached) + callee === ci && (ci = cached) + callee = cached + end end end push!(codeinsts, callee) @@ -1841,7 +1868,9 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; code_cache(interp)[mi] = callee else # Use an existing CI from the cache, if there is available one that is compatible - callee = cached + if link_ci_equiv!(callee, cached) + callee = cached + end end end push!(codeinfos, callee) diff --git a/src/gf.c b/src/gf.c index 2085beabc68a1..812cfd46256e6 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3786,6 +3786,60 @@ static int need_copy_to_mi_cache(jl_method_instance_t *mi, jl_method_instance_t !jl_egal((jl_value_t*)mi->sparam_vals, (jl_value_t*)mi2->sparam_vals); } +// Register `caller` in the backedges of each dependency in the forward edges svec. +// Mirrors the Julia-side store_backedges, but callable from C. +static void store_backedges_c(jl_code_instance_t *caller, jl_svec_t *edges) +{ + if (edges == NULL) + return; + jl_method_instance_t *caller_mi = jl_get_ci_mi(caller); + if (!jl_is_method(caller_mi->def.method)) + return; + size_t n = jl_svec_len(edges); + if (n == 0) + return; + for (size_t i = 0; i < n; ) { + jl_value_t *item = jl_svecref(edges, i); + if (jl_is_long(item)) { + i += 2; + continue; + } + if (jl_is_method(item)) { + i += 1; + continue; + } + jl_value_t *invokesig = NULL; + if (!jl_is_method_instance(item) && !jl_is_code_instance(item) && !jl_is_binding(item)) { + // it's a Type used as invokesig + invokesig = item; + i += 1; + if (i >= n) + break; + item = jl_svecref(edges, i); + if (jl_is_method(item)) { + i += 1; + continue; + } + if (jl_is_mtable(item)) { + jl_method_table_add_backedge(invokesig, caller); + i += 1; + continue; + } + } + if (jl_is_binding(item)) { + jl_maybe_add_binding_backedge((jl_binding_t*)item, (jl_value_t*)caller, caller_mi->def.method); + } + else if (jl_is_code_instance(item)) { + jl_method_instance_add_backedge(jl_get_ci_mi((jl_code_instance_t*)item), invokesig, caller); + } + else if (jl_is_method_instance(item)) { + jl_method_instance_add_backedge((jl_method_instance_t*)item, invokesig, caller); + } + i += 1; + } +} + + static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_code_instance_t *codeinst2) JL_CANSAFEPOINT { jl_code_instance_t *codeinst = jl_get_method_uninferred( @@ -3795,7 +3849,11 @@ static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGAT jl_atomic_load_relaxed(&codeinst2->debuginfo), jl_atomic_load_relaxed(&codeinst2->edges)); if (jl_atomic_load_relaxed(&codeinst->invoke) == NULL) { - // TODO: add edges and jl_promote_ci_to_current here + JL_GC_PUSH2(&codeinst, &codeinst2); + jl_svec_t *edges = jl_atomic_load_relaxed(&codeinst->edges); + store_backedges_c(codeinst, edges); + jl_promote_ci_to_current(codeinst, jl_get_world_counter()); + JL_GC_POP(); jl_gc_write(codeinst, codeinst->rettype_const, jl_value_t, codeinst2->rettype_const); uint8_t specsigflags; jl_callptr_t invoke; From 97ce87166c4cc4e40c07b0219a7d62987dbeeceb Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 16 Jun 2026 16:01:16 +0000 Subject: [PATCH 2/6] codegen: link CI equiv in findCompatibleCI and get_ci_equiv_compiled Extend the CI equivalence edge/backedge fix to the C-side sites where equivalent CodeInstances are found and substituted. Adds jl_link_ci_equiv, a C-level mirror of link_ci_equiv! from typeinfer.jl, and calls it from: - aot_link_output (get_ci_equiv_compiled): AOT call-target routing - linkOutput (findCompatibleCI): JIT call-target routing In both cases, when a callee CI is routed through a cached/equiv CI, jl_link_ci_equiv ensures the callee ends up in the equiv's MI backedges and that the equiv appears in the callee's forward edges. Without this, invalidation of the equiv would not propagate to the callee. The jl_link_ci_equiv call in linkOutput is made with LinkerMutex temporarily released, since jl_alloc_svec may GC. Co-Authored-By: Claude Sonnet 4.6 --- Compiler/src/typeinfer.jl | 25 +-------- src/aotcompile.cpp | 7 ++- src/ast.c | 2 +- src/gf.c | 104 ++++++++++++++++++-------------------- src/jitlayers.cpp | 52 +++++++++++++++---- src/jitlayers.h | 5 +- src/julia_internal.h | 4 +- 7 files changed, 105 insertions(+), 94 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 8d3eed5d84d8a..4cf5cd51a28ab 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -818,30 +818,7 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) nothing end -# When `callee` (not in the global MI cache) is being replaced by an equivalent `cached` -# (in the MI cache) for JIT compilation, ensure callee will be properly invalidated if -# cached is invalidated later. We do this by making callee depend on cached (adding cached -# to callee.edges) and registering callee in cached's MI backedges. -function link_ci_equiv!(callee::CodeInstance, cached::CodeInstance) - callee === cached && return true - mi = get_ci_mi(cached) - isa(mi.def, Method) || return true - # check if callee already has cached in its edges - old_edges = callee.edges::SimpleVector - for e in old_edges - e === cached && return true - end - validation_world = get_world_counter() - if callee.max_world > validation_world - # append cached to callee's edges so _invalidate_backedges can find it by identity - new_edges = Core.svec(old_edges..., cached) - @atomic :release callee.edges = new_edges - # register callee in cached's MI backedges so invalidation of cached propagates to callee - store_backedges(callee, Core.svec(cached)) - end - # after storing new backedges, use method if it is still legal - return cached.max_world > callee.max_world -end +link_ci_equiv!(callee::CodeInstance, cached::CodeInstance) = !iszero(ccall(:jl_link_ci_equiv, Cint, (Any, Any), callee, cached)) function compute_edges!(sv::InferenceState) edges = sv.edges diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 5de83829aaf1a..9fe6d02fcb896 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -847,8 +847,11 @@ static void aot_link_output(jl_codegen_output_t &out) JL_CANSAFEPOINT continue; auto it = out.ci_funcs.find(ci); - if (it == out.ci_funcs.end()) - it = get_ci_equiv_compiled(ci, out.ci_funcs); + if (it == out.ci_funcs.end()) { + auto equiv = get_ci_equiv_compiled(ci, out.ci_funcs); + if (equiv != out.ci_funcs.end() && jl_link_ci_equiv(ci, equiv->first)) + it = equiv; + } jl_codeinst_funcs_t funcs; if (it != out.ci_funcs.end()) { funcs = {it->second.invoke_api, it->second.invoke, it->second.specptr}; diff --git a/src/ast.c b/src/ast.c index eb6fc3f21e8fc..5a6867ae33f5d 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1072,7 +1072,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule jl_timing_show_macro(mfunc, retry_lno != NULL ? retry_lno : margs[1], inmodule, JL_TIMING_DEFAULT_BLOCK); *ctx = mfunc->def.method->module; - result = jl_invoke(margs[0], &margs[1], nargs - 1, mfunc); + result = jl_invoke_fromdispatch(margs[0], &margs[1], nargs - 1, mfunc); } JL_CATCH { if ((jl_loaderror_type == NULL) || !throw_load_error) { diff --git a/src/gf.c b/src/gf.c index 812cfd46256e6..c5b892df26f61 100644 --- a/src/gf.c +++ b/src/gf.c @@ -586,8 +586,8 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_uninferred( jl_value_t *owner = jl_nothing; // TODO: owner should be arg jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { - if (jl_atomic_load_relaxed(&codeinst->min_world) == min_world && - jl_atomic_load_relaxed(&codeinst->max_world) == max_world && + if (jl_atomic_load_relaxed(&codeinst->min_world) <= min_world && + jl_atomic_load_relaxed(&codeinst->max_world) >= max_world && jl_egal(codeinst->owner, owner) && jl_egal(codeinst->rettype, rettype)) { if (di == NULL) @@ -3786,74 +3786,64 @@ static int need_copy_to_mi_cache(jl_method_instance_t *mi, jl_method_instance_t !jl_egal((jl_value_t*)mi->sparam_vals, (jl_value_t*)mi2->sparam_vals); } -// Register `caller` in the backedges of each dependency in the forward edges svec. -// Mirrors the Julia-side store_backedges, but callable from C. -static void store_backedges_c(jl_code_instance_t *caller, jl_svec_t *edges) +// Link callee to cached so that invalidation of cached propagates to callee. +// When a CodeInstance swap occurs (callee replaced by cached during compilation), +// we must ensure callee gets invalidated whenever cached is, since any caller +// holding callee by pointer identity in its edges won't otherwise be reached. +// Return 1 if cached should replace callee (cached covers a wider world range), +// or 0 if cached object is no longer current (world counter invalidated it concurrently). +// Sets up edges and backedges so that invalidation of cached propagates to callee. +JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t *cached) { - if (edges == NULL) - return; - jl_method_instance_t *caller_mi = jl_get_ci_mi(caller); - if (!jl_is_method(caller_mi->def.method)) - return; - size_t n = jl_svec_len(edges); - if (n == 0) - return; - for (size_t i = 0; i < n; ) { - jl_value_t *item = jl_svecref(edges, i); - if (jl_is_long(item)) { - i += 2; - continue; - } - if (jl_is_method(item)) { - i += 1; - continue; - } - jl_value_t *invokesig = NULL; - if (!jl_is_method_instance(item) && !jl_is_code_instance(item) && !jl_is_binding(item)) { - // it's a Type used as invokesig - invokesig = item; - i += 1; - if (i >= n) + if (callee == cached) + return 1; + jl_method_instance_t *mi = jl_get_ci_mi(cached); + if (!jl_is_method(mi->def.method)) + return 1; + size_t world = jl_get_world_counter(); + if (jl_atomic_load_relaxed(&callee->max_world) >= world) { + // Append cached to callee->edges so _invalidate_backedges can find it by identity + int already_linked = 0; + jl_svec_t *old_edges = jl_atomic_load_relaxed(&callee->edges); + size_t old_n = jl_svec_len(old_edges); + for (size_t i = 0; i < old_n; i++) { + if (jl_svecref(old_edges, i) == (jl_value_t *)cached) { + already_linked = 1; break; - item = jl_svecref(edges, i); - if (jl_is_method(item)) { - i += 1; - continue; } - if (jl_is_mtable(item)) { - jl_method_table_add_backedge(invokesig, caller); - i += 1; - continue; - } - } - if (jl_is_binding(item)) { - jl_maybe_add_binding_backedge((jl_binding_t*)item, (jl_value_t*)caller, caller_mi->def.method); } - else if (jl_is_code_instance(item)) { - jl_method_instance_add_backedge(jl_get_ci_mi((jl_code_instance_t*)item), invokesig, caller); + if (!already_linked) { + jl_svec_t *new_edges = jl_alloc_svec(old_n + 1); + memcpy(jl_svec_data(new_edges), jl_svec_data(old_edges), old_n * sizeof(jl_value_t *)); + jl_svecset(new_edges, old_n, (jl_value_t *)cached); + // an invoke-type edge would be more accurate here, but this is conservatively correct + jl_gc_write_atomic(callee, callee->edges, jl_svec_t, new_edges, release); + // register callee in cached's MI backedges so invalidation of cached propagates to callee + jl_method_instance_add_backedge(mi, NULL, callee); } - else if (jl_is_method_instance(item)) { - jl_method_instance_add_backedge((jl_method_instance_t*)item, invokesig, caller); - } - i += 1; } + // after storing new backedges, use cached if it is still legal (covers wider world range) + return jl_atomic_load_relaxed(&cached->max_world) >= jl_atomic_load_relaxed(&callee->max_world); } static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_code_instance_t *codeinst2) JL_CANSAFEPOINT { + size_t current_world = jl_get_world_counter(); + size_t max_world2 = jl_atomic_load_relaxed(&codeinst2->max_world); jl_code_instance_t *codeinst = jl_get_method_uninferred( mi, codeinst2->rettype, jl_atomic_load_relaxed(&codeinst2->min_world), - jl_atomic_load_relaxed(&codeinst2->max_world), // TODO: use min(max_world, current_world) here + max_world2 < current_world ? max_world2 : current_world, jl_atomic_load_relaxed(&codeinst2->debuginfo), jl_atomic_load_relaxed(&codeinst2->edges)); if (jl_atomic_load_relaxed(&codeinst->invoke) == NULL) { - JL_GC_PUSH2(&codeinst, &codeinst2); - jl_svec_t *edges = jl_atomic_load_relaxed(&codeinst->edges); - store_backedges_c(codeinst, edges); - jl_promote_ci_to_current(codeinst, jl_get_world_counter()); - JL_GC_POP(); + // if codeinst2 is still valid beyond current_world, link codeinst to it so + // that invalidation of codeinst2 also invalidates codeinst + if (max_world2 == ~(size_t)0) { + if (jl_link_ci_equiv(codeinst, codeinst2)) + jl_promote_ci_to_current(codeinst, current_world); + } jl_gc_write(codeinst, codeinst->rettype_const, jl_value_t, codeinst2->rettype_const); uint8_t specsigflags; jl_callptr_t invoke; @@ -4545,10 +4535,16 @@ JL_DLLEXPORT jl_value_t *jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t na return _jl_invoke(F, args, nargs, mfunc, world, TRIGGER_FOREIGN); } +jl_value_t *jl_invoke_fromdispatch(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc) +{ + size_t world = jl_current_task->world_age; + return _jl_invoke(F, args, nargs, mfunc, world, TRIGGER_DISPATCH); +} + // Used by jl_eval_thunk to invoke top-level thunks. They will be // garbage-collectable as soon as they are invoked, so their ORC symbols must be // unregistered before we enter invoke, which may never return. -JL_DLLEXPORT jl_value_t *jl_invoke_oneshot(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc) +jl_value_t *jl_invoke_oneshot(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc) { size_t world = jl_current_task->world_age; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 1ad2a788d4472..bddb4a5ba9bcf 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2301,7 +2301,9 @@ renameLinkGraphSymbol(jitlink::LinkGraph &G, jitlink::Symbol &Sym, bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferRef ObjBuf, jitlink::LinkGraph &G, std::unique_ptr Info) + JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { + jl_task_t *ct = jl_current_task; std::unique_lock Lock{LinkerMutex}; auto Syms = linkGraphSymbols(G); @@ -2324,6 +2326,33 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR RenameDef(Funcs.specptr, S.specptr); } + // Pre-pass: find CI equivalents, call jl_link_ci_equiv for each, and build + // EquivMap for use in the main pass to avoid re-running findCompatibleCI. + // This condition should match that in jl_add_codeinst_to_jit!, which will + // add a different, compatible CodeInstance to the JIT but not update the + // invoke statement. + DenseMap EquivMap; + for (auto &[Call, T] : Info->call_targets) { + auto [CI, API] = Call; + JL_GC_PROMISE_ROOTED(CI); + if (!Syms.contains(T)) + continue; + if (EquivMap.contains(CI)) + continue; + if (!jl_mi_cache_has_ci(jl_get_ci_mi(CI), CI)) { + jl_code_instance_t *Equiv = findCompatibleCI(CI); + if (Equiv != CI) { + Lock.unlock(); + int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); + bool use_equiv = jl_link_ci_equiv(CI, Equiv); + jl_gc_unsafe_leave(ct->ptls, gc_state); + Lock.lock(); + if (use_equiv) + EquivMap[CI] = Equiv; + } + } + } + // Rename referenced CIs in the workqueue. for (auto &[Call, T] : Info->call_targets) { auto [CI, API] = Call; @@ -2339,7 +2368,7 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR continue; } JL_GC_PROMISE_ROOTED(CI); - auto Dest = linkCallTarget(MR, CI, API); + auto Dest = linkCallTarget(MR, CI, API, EquivMap); if (!Dest) return false; if (auto *DestSym = findLinkGraphSymbolByName(G, Dest); @@ -2398,16 +2427,19 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR // Must hold LinkerMutex. orc::SymbolStringPtr JuliaOJIT::linkCallTarget(orc::MaterializationResponsibility &MR, - jl_code_instance_t *CI, jl_invoke_api_t API) + jl_code_instance_t *CI, jl_invoke_api_t API, + const DenseMap &EquivMap) { - // This condition should match that in jl_add_codeinst_to_jit!, which will - // add a different, compatible CodeInstance to the JIT but not update the - // invoke statement. - if (!jl_mi_cache_has_ci(jl_get_ci_mi(CI), CI)) - CI = findCompatibleCI(CI); - auto It = CISymbols.find(CI); - if (It != CISymbols.end() && It->second.invoke_api == API) - return It->second.specptr; + { + auto It = EquivMap.find(CI); + if (It != EquivMap.end()) + CI = It->second; + } + { + auto It = CISymbols.find(CI); + if (It != CISymbols.end() && It->second.invoke_api == API) + return It->second.specptr; + } CISymbolPtr *Sym = linkCISymbol(CI); diff --git a/src/jitlayers.h b/src/jitlayers.h index d0b34155f881e..cb9e86b99909c 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -890,7 +890,7 @@ class JuliaOJIT { // false after calling MR.failMaterialization(). bool linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferRef ObjBuf, jitlink::LinkGraph &G, - std::unique_ptr Info) JL_NOTSAFEPOINT; + std::unique_ptr Info) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER; // Return a symbol that should be linked to the call target. The origin of // this symbol depends on the code instance: @@ -904,7 +904,8 @@ class JuliaOJIT { // new module and return a symbol for it. orc::SymbolStringPtr linkCallTarget(orc::MaterializationResponsibility &MR, jl_code_instance_t *CI, - jl_invoke_api_t API) JL_NOTSAFEPOINT; + jl_invoke_api_t API, + const DenseMap &EquivMap) JL_NOTSAFEPOINT; // If the provided CodeInstance is neither compiled nor has an ORC symbol in // CISymbols, look for a compatible CodeInstance in the MethodInstance's diff --git a/src/julia_internal.h b/src/julia_internal.h index 7c2881a2398f3..5376787d2ad7b 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -826,7 +826,8 @@ JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, jl_code_instance_t JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_CANSAFEPOINT; JL_DLLEXPORT void jl_add_codeinsts_to_jit(jl_array_t *codeinsts, jl_array_t *srcs) JL_CANSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_invoke_oneshot(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *meth) JL_CANSAFEPOINT; +jl_value_t *jl_invoke_oneshot(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *meth) JL_CANSAFEPOINT; +jl_value_t *jl_invoke_fromdispatch(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc) JL_CANSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner) JL_CANSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( @@ -837,6 +838,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( uint32_t effects, jl_value_t *analysis_results, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/) JL_CANSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT; +JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t *cached); STATIC_INLINE jl_method_instance_t *jl_get_ci_mi(jl_code_instance_t *ci JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { From 1a50d73510c654629e7feac73f6df7edcc135107 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 29 Jun 2026 17:15:44 +0000 Subject: [PATCH 3/6] redesign --- Compiler/src/typeinfer.jl | 20 ++++++++---------- src/aotcompile.cpp | 6 ++++-- src/gf.c | 43 +++++++++++++++++++++++++++------------ src/jitlayers.cpp | 4 +--- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 4cf5cd51a28ab..d910aeb071a40 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -818,8 +818,6 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) nothing end -link_ci_equiv!(callee::CodeInstance, cached::CodeInstance) = !iszero(ccall(:jl_link_ci_equiv, Cint, (Any, Any), callee, cached)) - function compute_edges!(sv::InferenceState) edges = sv.edges for i in 1:length(sv.stmt_info) @@ -1725,6 +1723,10 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn markinspected!(workqueue, callee) continue end + let cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance + markinspected!(workqueue, callee) + continue + end src = ci_get_source(interp, callee) if !isa(src, CodeInfo) newcallee = typeinf_ext(workqueue.interp, callee.def, source_mode) # always SOURCE_MODE_ABI @@ -1742,16 +1744,14 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn sptypes = sptypes_from_meth_instance(mi) collectinvokes!(workqueue, src, sptypes) if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, 0x0)::CodeInstance if cached === callee # make sure callee is gc-rooted and cached, as required by jl_add_codeinsts_to_jit code_cache(workqueue.interp)[mi] = callee else # use an existing CI from the cache, if there is available one that is compatible - if link_ci_equiv!(callee, cached) - callee === ci && (ci = cached) - callee = cached - end + callee === ci && (ci = cached) + callee = cached end end push!(codeinsts, callee) @@ -1840,14 +1840,12 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; enqueue_unprepared_invokes) # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, world)::CodeInstance + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, 0x0)::CodeInstance if cached === callee code_cache(interp)[mi] = callee else # Use an existing CI from the cache, if there is available one that is compatible - if link_ci_equiv!(callee, cached) - callee = cached - end + callee = cached end end push!(codeinfos, callee) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 9fe6d02fcb896..a7d7dee6a59c0 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -849,8 +849,10 @@ static void aot_link_output(jl_codegen_output_t &out) JL_CANSAFEPOINT auto it = out.ci_funcs.find(ci); if (it == out.ci_funcs.end()) { auto equiv = get_ci_equiv_compiled(ci, out.ci_funcs); - if (equiv != out.ci_funcs.end() && jl_link_ci_equiv(ci, equiv->first)) - it = equiv; + if (equiv != out.ci_funcs.end()) + // If ci is subject to invalidation, link it against equiv + if (jl_link_ci_equiv(ci, equiv->first)) + it = equiv; } jl_codeinst_funcs_t funcs; if (it != out.ci_funcs.end()) { diff --git a/src/gf.c b/src/gf.c index c5b892df26f61..2f4c1aa89c804 100644 --- a/src/gf.c +++ b/src/gf.c @@ -624,23 +624,29 @@ JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, return 0; } -// look for something with an egal ABI and properties that is already in the JIT for a whole edge (target_world=0) or can be added to the JIT with new source just for target_world. +// look for something with an egal ABI and properties that is already in the JIT for the target_world, or could be added to the JIT instead of ci to satisfy the same invoke edge with the same src. JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT { jl_value_t *def = ci->def; jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_value_t *owner = ci->owner; jl_value_t *rettype = ci->rettype; + size_t min_world = target_world ? target_world : jl_atomic_load_relaxed(&ci->min_world); + size_t max_world = target_world ? target_world : jl_atomic_load_relaxed(&ci->max_world); jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); while (codeinst) { if (codeinst != ci && jl_atomic_load_relaxed(&codeinst->inferred) != NULL && - jl_atomic_load_relaxed(&codeinst->min_world) <= target_world && - jl_atomic_load_relaxed(&codeinst->max_world) >= target_world && jl_egal(codeinst->def, def) && jl_egal(codeinst->owner, owner) && jl_egal(codeinst->rettype, rettype)) { - return codeinst; + if (!target_world || jl_atomic_load_relaxed(&codeinst->invoke) != NULL) { + size_t min_world2 = jl_atomic_load_relaxed(&codeinst->min_world); + size_t max_world2 = jl_atomic_load_relaxed(&codeinst->max_world); + if (target_world ? (min_world2 == min_world && max_world2 == max_world) : + (min_world2 <= min_world && max_world2 >= max_world)) + return codeinst; + } } codeinst = jl_atomic_load_relaxed(&codeinst->next); } @@ -3797,11 +3803,15 @@ JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t { if (callee == cached) return 1; + if (jl_atomic_load_relaxed(&callee->max_world) != ~(size_t)0) + return 1; jl_method_instance_t *mi = jl_get_ci_mi(cached); if (!jl_is_method(mi->def.method)) return 1; - size_t world = jl_get_world_counter(); - if (jl_atomic_load_relaxed(&callee->max_world) >= world) { + // Take a lock here to simplify the ordering needed here by ensuring worlds and edges don't change during these steps + JL_LOCK(&world_counter_lock); + int add_edge = jl_atomic_load_relaxed(&cached->max_world) == ~(size_t)0; + if (add_edge) { // Append cached to callee->edges so _invalidate_backedges can find it by identity int already_linked = 0; jl_svec_t *old_edges = jl_atomic_load_relaxed(&callee->edges); @@ -3822,8 +3832,8 @@ JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t jl_method_instance_add_backedge(mi, NULL, callee); } } - // after storing new backedges, use cached if it is still legal (covers wider world range) - return jl_atomic_load_relaxed(&cached->max_world) >= jl_atomic_load_relaxed(&callee->max_world); + JL_UNLOCK(&world_counter_lock); + return add_edge; } @@ -3831,18 +3841,25 @@ static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGAT { size_t current_world = jl_get_world_counter(); size_t max_world2 = jl_atomic_load_relaxed(&codeinst2->max_world); + // if codeinst2 is still valid beyond current_world, link codeinst to + // it so that invalidation of codeinst2 also invalidates codeinst + jl_svec_t *copy_edge = jl_svec1(codeinst2); + // TODO: this should have been an invoke edge: + //jl_svec_t *copy_edge = jl_svec2(mi->sig, codeinst2); jl_code_instance_t *codeinst = jl_get_method_uninferred( mi, codeinst2->rettype, jl_atomic_load_relaxed(&codeinst2->min_world), max_world2 < current_world ? max_world2 : current_world, jl_atomic_load_relaxed(&codeinst2->debuginfo), - jl_atomic_load_relaxed(&codeinst2->edges)); + copy_edge); if (jl_atomic_load_relaxed(&codeinst->invoke) == NULL) { - // if codeinst2 is still valid beyond current_world, link codeinst to it so - // that invalidation of codeinst2 also invalidates codeinst if (max_world2 == ~(size_t)0) { - if (jl_link_ci_equiv(codeinst, codeinst2)) - jl_promote_ci_to_current(codeinst, current_world); + JL_LOCK(&world_counter_lock); + if (jl_atomic_load_relaxed(&codeinst2->max_world) == ~(size_t)0) { + jl_method_instance_add_backedge(mi, NULL, codeinst); + jl_atomic_store_relaxed(&codeinst->max_world, ~(size_t)0); // jl_promote_ci_to_current + } + JL_UNLOCK(&world_counter_lock); } jl_gc_write(codeinst, codeinst->rettype_const, jl_value_t, codeinst2->rettype_const); uint8_t specsigflags; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index bddb4a5ba9bcf..c6ab89ecb0664 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2328,9 +2328,6 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR // Pre-pass: find CI equivalents, call jl_link_ci_equiv for each, and build // EquivMap for use in the main pass to avoid re-running findCompatibleCI. - // This condition should match that in jl_add_codeinst_to_jit!, which will - // add a different, compatible CodeInstance to the JIT but not update the - // invoke statement. DenseMap EquivMap; for (auto &[Call, T] : Info->call_targets) { auto [CI, API] = Call; @@ -2344,6 +2341,7 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR if (Equiv != CI) { Lock.unlock(); int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); + // If ci is subject to invalidation, link it against equiv bool use_equiv = jl_link_ci_equiv(CI, Equiv); jl_gc_unsafe_leave(ct->ptls, gc_state); Lock.lock(); From 6707abed7ecba76df49bd9607505c40a5054769a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 29 Jun 2026 19:34:58 +0000 Subject: [PATCH 4/6] finish redesign warning: this make sysimage 160 -> 190 MB, which needs fixing (due to bad edges list) --- Compiler/src/typeinfer.jl | 36 +++++----- src/aotcompile.cpp | 18 +---- src/gf.c | 134 +++++++++++++++++++++----------------- src/jitlayers.cpp | 46 ++++--------- src/julia_internal.h | 3 +- 5 files changed, 112 insertions(+), 125 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index d910aeb071a40..3ac1d0b2ab9e2 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1724,8 +1724,10 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn continue end let cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance - markinspected!(workqueue, callee) - continue + if cached !== callee + markinspected!(workqueue, callee) + continue + end end src = ci_get_source(interp, callee) if !isa(src, CodeInfo) @@ -1744,14 +1746,15 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn sptypes = sptypes_from_meth_instance(mi) collectinvokes!(workqueue, src, sptypes) if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, 0x0)::CodeInstance - if cached === callee - # make sure callee is gc-rooted and cached, as required by jl_add_codeinsts_to_jit - code_cache(workqueue.interp)[mi] = callee - else - # use an existing CI from the cache, if there is available one that is compatible - callee === ci && (ci = cached) - callee = cached + let cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, 0x0)::CodeInstance + if cached === callee + # make sure callee is gc-rooted and cached, as required by jl_add_codeinsts_to_jit + code_cache(workqueue.interp)[mi] = callee + else + # use an existing CI from the cache, if there is available one that is compatible + callee === ci && (ci = cached) + callee = cached + end end end push!(codeinsts, callee) @@ -1840,12 +1843,13 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; enqueue_unprepared_invokes) # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, 0x0)::CodeInstance - if cached === callee - code_cache(interp)[mi] = callee - else - # Use an existing CI from the cache, if there is available one that is compatible - callee = cached + let cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, 0x0)::CodeInstance + if cached === callee + code_cache(interp)[mi] = callee + else + # Use an existing CI from the cache, if there is available one that is compatible + callee = cached + end end end push!(codeinfos, callee) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index a7d7dee6a59c0..5d594809fb02d 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -816,22 +816,10 @@ static Function *emit_pkg_plt_thunk(jl_codegen_output_t &out, jl_code_instance_t static jl_compiled_functions_t::iterator get_ci_equiv_compiled(jl_code_instance_t *ci JL_PROPAGATES_ROOT, jl_compiled_functions_t &compiled_functions) JL_NOTSAFEPOINT { - jl_value_t *def = ci->def; - jl_value_t *owner = ci->owner; - jl_value_t *rettype = ci->rettype; - size_t min_world = jl_atomic_load_relaxed(&ci->min_world); - size_t max_world = jl_atomic_load_relaxed(&ci->max_world); for (auto it = compiled_functions.begin(), E = compiled_functions.end(); it != E; ++it) { auto codeinst = it->first; - if (codeinst != ci && - jl_atomic_load_relaxed(&codeinst->inferred) != NULL && - jl_atomic_load_relaxed(&codeinst->min_world) <= min_world && - jl_atomic_load_relaxed(&codeinst->max_world) >= max_world && - jl_egal(codeinst->def, def) && - jl_egal(codeinst->owner, owner) && - jl_egal(codeinst->rettype, rettype)) { + if (codeinst != ci && jl_is_ci_equiv(ci, codeinst, 0)) return it; - } } return compiled_functions.end(); } @@ -850,9 +838,7 @@ static void aot_link_output(jl_codegen_output_t &out) JL_CANSAFEPOINT if (it == out.ci_funcs.end()) { auto equiv = get_ci_equiv_compiled(ci, out.ci_funcs); if (equiv != out.ci_funcs.end()) - // If ci is subject to invalidation, link it against equiv - if (jl_link_ci_equiv(ci, equiv->first)) - it = equiv; + it = equiv; } jl_codeinst_funcs_t funcs; if (it != out.ci_funcs.end()) { diff --git a/src/gf.c b/src/gf.c index 2f4c1aa89c804..db6c10efc958a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -599,7 +599,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_uninferred( if (!(debuginfo && jl_egal((jl_value_t*)debuginfo, (jl_value_t*)di))) continue; } - // TODO: this is implied by the matching worlds, since it is intrinsic, so do we really need to verify it? + // n.b.: this is implied by the matching worlds for min_world, but not max_world == ~0 (which couldn't compute this) jl_svec_t *e = jl_atomic_load_relaxed(&codeinst->edges); if (e && jl_egal((jl_value_t*)e, (jl_value_t*)edges)) return codeinst; @@ -624,30 +624,49 @@ JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, return 0; } -// look for something with an egal ABI and properties that is already in the JIT for the target_world, or could be added to the JIT instead of ci to satisfy the same invoke edge with the same src. -JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT +// return whether the ci has more restrictions than the other arguments (more edges and narrower worlds) +static int jl_codeinst_edges_sub(jl_code_instance_t *ci, size_t min_world2, size_t max_world2, jl_svec_t *edges2) +{ + size_t min_world = jl_atomic_load_relaxed(&ci->min_world); + size_t max_world = jl_atomic_load_relaxed(&ci->max_world); + jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); + // n.b.: edges matching is implied sufficiently by the matching worlds for min_world, but not max_world == ~0 (which couldn't compute that) + return min_world >= min_world2 && max_world <= max_world2 && edges && jl_egal((jl_value_t*)edges, (jl_value_t*)edges2); +} + +// return whether the codeinst can be substituted in place of ci for an invoke target in target_world +JL_DLLEXPORT int jl_is_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, jl_code_instance_t *codeinst, size_t target_world) JL_NOTSAFEPOINT { jl_value_t *def = ci->def; - jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_value_t *owner = ci->owner; jl_value_t *rettype = ci->rettype; - size_t min_world = target_world ? target_world : jl_atomic_load_relaxed(&ci->min_world); - size_t max_world = target_world ? target_world : jl_atomic_load_relaxed(&ci->max_world); - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); - while (codeinst) { - if (codeinst != ci && - jl_atomic_load_relaxed(&codeinst->inferred) != NULL && - jl_egal(codeinst->def, def) && - jl_egal(codeinst->owner, owner) && - jl_egal(codeinst->rettype, rettype)) { - if (!target_world || jl_atomic_load_relaxed(&codeinst->invoke) != NULL) { - size_t min_world2 = jl_atomic_load_relaxed(&codeinst->min_world); - size_t max_world2 = jl_atomic_load_relaxed(&codeinst->max_world); - if (target_world ? (min_world2 == min_world && max_world2 == max_world) : - (min_world2 <= min_world && max_world2 >= max_world)) - return codeinst; + if (jl_egal(codeinst->def, def) && + jl_egal(codeinst->owner, owner) && + jl_egal(codeinst->rettype, rettype)) { + if (!target_world || jl_atomic_load_relaxed(&codeinst->invoke) != NULL) { + size_t min_world = jl_atomic_load_relaxed(&ci->min_world); + size_t max_world = jl_atomic_load_relaxed(&ci->max_world); + size_t min_world2 = jl_atomic_load_relaxed(&codeinst->min_world); + size_t max_world2 = jl_atomic_load_relaxed(&codeinst->max_world); + if (target_world || (min_world2 == min_world && max_world2 == max_world)) { + jl_svec_t *edges2 = jl_atomic_load_relaxed(&codeinst->edges); + if (jl_codeinst_edges_sub(ci, min_world2, max_world2, edges2)) { + return 1; + } } } + } + return 0; +} + +// look for something with an egal ABI and properties that is already in the JIT for the target_world, or could be added to the JIT instead of ci to satisfy the same invoke edge with the same src. +JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT +{ + jl_method_instance_t *mi = jl_get_ci_mi(ci); + jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); + while (codeinst) { + if (codeinst != ci && jl_is_ci_equiv(ci, codeinst, target_world)) + return codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next); } return ci; @@ -3799,42 +3818,42 @@ static int need_copy_to_mi_cache(jl_method_instance_t *mi, jl_method_instance_t // Return 1 if cached should replace callee (cached covers a wider world range), // or 0 if cached object is no longer current (world counter invalidated it concurrently). // Sets up edges and backedges so that invalidation of cached propagates to callee. -JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t *cached) -{ - if (callee == cached) - return 1; - if (jl_atomic_load_relaxed(&callee->max_world) != ~(size_t)0) - return 1; - jl_method_instance_t *mi = jl_get_ci_mi(cached); - if (!jl_is_method(mi->def.method)) - return 1; - // Take a lock here to simplify the ordering needed here by ensuring worlds and edges don't change during these steps - JL_LOCK(&world_counter_lock); - int add_edge = jl_atomic_load_relaxed(&cached->max_world) == ~(size_t)0; - if (add_edge) { - // Append cached to callee->edges so _invalidate_backedges can find it by identity - int already_linked = 0; - jl_svec_t *old_edges = jl_atomic_load_relaxed(&callee->edges); - size_t old_n = jl_svec_len(old_edges); - for (size_t i = 0; i < old_n; i++) { - if (jl_svecref(old_edges, i) == (jl_value_t *)cached) { - already_linked = 1; - break; - } - } - if (!already_linked) { - jl_svec_t *new_edges = jl_alloc_svec(old_n + 1); - memcpy(jl_svec_data(new_edges), jl_svec_data(old_edges), old_n * sizeof(jl_value_t *)); - jl_svecset(new_edges, old_n, (jl_value_t *)cached); - // an invoke-type edge would be more accurate here, but this is conservatively correct - jl_gc_write_atomic(callee, callee->edges, jl_svec_t, new_edges, release); - // register callee in cached's MI backedges so invalidation of cached propagates to callee - jl_method_instance_add_backedge(mi, NULL, callee); - } - } - JL_UNLOCK(&world_counter_lock); - return add_edge; -} +//JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t *cached) +//{ +// if (callee == cached) +// return 1; +// if (jl_atomic_load_relaxed(&callee->max_world) != ~(size_t)0) +// return 1; +// jl_method_instance_t *mi = jl_get_ci_mi(cached); +// if (!jl_is_method(mi->def.method)) +// return 1; +// // Take a lock here to simplify the ordering needed here by ensuring worlds and edges don't change during these steps +// JL_LOCK(&world_counter_lock); +// int add_edge = jl_atomic_load_relaxed(&cached->max_world) == ~(size_t)0; +// if (add_edge) { +// // Append cached to callee->edges so _invalidate_backedges can find it by identity +// int already_linked = 0; +// jl_svec_t *old_edges = jl_atomic_load_relaxed(&callee->edges); +// size_t old_n = jl_svec_len(old_edges); +// for (size_t i = 0; i < old_n; i++) { +// if (jl_svecref(old_edges, i) == (jl_value_t *)cached) { +// already_linked = 1; +// break; +// } +// } +// if (!already_linked) { +// jl_svec_t *new_edges = jl_alloc_svec(old_n + 1); +// memcpy(jl_svec_data(new_edges), jl_svec_data(old_edges), old_n * sizeof(jl_value_t *)); +// jl_svecset(new_edges, old_n, (jl_value_t *)cached); +// // an invoke-type edge would be more accurate here, but this is conservatively correct +// jl_gc_write_atomic(callee, callee->edges, jl_svec_t, new_edges, release); +// // register callee in cached's MI backedges so invalidation of cached propagates to callee +// jl_method_instance_add_backedge(mi, NULL, callee); +// } +// } +// JL_UNLOCK(&world_counter_lock); +// return add_edge; +//} static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_code_instance_t *codeinst2) JL_CANSAFEPOINT @@ -3843,9 +3862,8 @@ static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGAT size_t max_world2 = jl_atomic_load_relaxed(&codeinst2->max_world); // if codeinst2 is still valid beyond current_world, link codeinst to // it so that invalidation of codeinst2 also invalidates codeinst - jl_svec_t *copy_edge = jl_svec1(codeinst2); - // TODO: this should have been an invoke edge: - //jl_svec_t *copy_edge = jl_svec2(mi->sig, codeinst2); + jl_method_t *m = mi->def.method; + jl_svec_t *copy_edge = jl_is_method(m) ? jl_svec2(m->sig, codeinst2) : jl_emptysvec; jl_code_instance_t *codeinst = jl_get_method_uninferred( mi, codeinst2->rettype, jl_atomic_load_relaxed(&codeinst2->min_world), diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index c6ab89ecb0664..865d60f181c19 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2303,7 +2303,6 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR jitlink::LinkGraph &G, std::unique_ptr Info) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { - jl_task_t *ct = jl_current_task; std::unique_lock Lock{LinkerMutex}; auto Syms = linkGraphSymbols(G); @@ -2326,8 +2325,8 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR RenameDef(Funcs.specptr, S.specptr); } - // Pre-pass: find CI equivalents, call jl_link_ci_equiv for each, and build - // EquivMap for use in the main pass to avoid re-running findCompatibleCI. + // Pre-pass: find CI equivalents, and build EquivMap for use in the main + // pass to memoize findCompatibleCI. DenseMap EquivMap; for (auto &[Call, T] : Info->call_targets) { auto [CI, API] = Call; @@ -2338,16 +2337,8 @@ bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferR continue; if (!jl_mi_cache_has_ci(jl_get_ci_mi(CI), CI)) { jl_code_instance_t *Equiv = findCompatibleCI(CI); - if (Equiv != CI) { - Lock.unlock(); - int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); - // If ci is subject to invalidation, link it against equiv - bool use_equiv = jl_link_ci_equiv(CI, Equiv); - jl_gc_unsafe_leave(ct->ptls, gc_state); - Lock.lock(); - if (use_equiv) - EquivMap[CI] = Equiv; - } + if (Equiv != CI) + EquivMap[CI] = Equiv; } } @@ -2468,30 +2459,19 @@ orc::SymbolStringPtr JuliaOJIT::linkCallTarget(orc::MaterializationResponsibilit return Sym->specptr; } -jl_code_instance_t *JuliaOJIT::findCompatibleCI(jl_code_instance_t *CI) +jl_code_instance_t *JuliaOJIT::findCompatibleCI(jl_code_instance_t *ci) { - // add_codeinsts_to_jit! may have added an equivalent CI to the JIT, but + // add_codeinsts_to_jit! may have added an equivalent ci to the JIT, but // the invoke itself won't be updated. - auto MI = jl_get_ci_mi(CI); - jl_value_t *Def = CI->def; - jl_value_t *Owner = CI->owner; - jl_value_t *RetType = CI->rettype; - size_t MinWorld = jl_atomic_load_relaxed(&CI->min_world); - size_t MaxWorld = jl_atomic_load_relaxed(&CI->max_world); - auto IsCompatible = [=](jl_code_instance_t *CI2) JL_NOTSAFEPOINT { - return jl_atomic_load_relaxed(&CI2->min_world) <= MinWorld && - jl_atomic_load_relaxed(&CI2->max_world) >= MaxWorld && - jl_egal(CI2->def, Def) && jl_egal(CI2->owner, Owner) && - jl_egal(CI2->rettype, RetType); - }; - for (auto CI2 = jl_atomic_load_relaxed(&MI->cache); CI2; - CI2 = jl_atomic_load_relaxed(&CI2->next)) { - if (CI2 != CI && IsCompatible(CI2) && - (CISymbols.contains(CI2) || jl_atomic_load_relaxed(&CI2->invoke))) { - return CI2; + auto mi = jl_get_ci_mi(ci); + for (auto ci2 = jl_atomic_load_relaxed(&mi->cache); ci2; + ci2 = jl_atomic_load_relaxed(&ci2->next)) { + if (ci2 != ci && jl_is_ci_equiv(ci, ci2, 0) && + (CISymbols.contains(ci2) || jl_atomic_load_relaxed(&ci2->invoke))) { + return ci2; } } - return CI; + return ci; } CISymbolPtr *JuliaOJIT::linkCISymbol(jl_code_instance_t *CI) diff --git a/src/julia_internal.h b/src/julia_internal.h index 5376787d2ad7b..aae67349e9713 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -838,8 +838,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( uint32_t effects, jl_value_t *analysis_results, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/) JL_CANSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT; -JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t *cached); - +JL_DLLEXPORT int jl_is_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, jl_code_instance_t *codeinst, size_t target_world) JL_NOTSAFEPOINT; STATIC_INLINE jl_method_instance_t *jl_get_ci_mi(jl_code_instance_t *ci JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { jl_value_t *def = ci->def; From 4feecbcdff2210ed379304588c26107dc2b5fd11 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 8 Jul 2026 19:12:09 +0000 Subject: [PATCH 5/6] defer real fix --- Compiler/src/typeinfer.jl | 2 +- src/gf.c | 47 +-------------------------------------- src/jitlayers.cpp | 1 - 3 files changed, 2 insertions(+), 48 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 3ac1d0b2ab9e2..cb1090adbc201 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1960,7 +1960,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m ci = ci::CodeInstance mi = get_ci_mi(ci) return !iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, ci)) || - ccall(:jl_get_ci_equiv, Any, (Any, UInt), ci, ci.min_world)::CodeInstance !== ci + ccall(:jl_get_ci_equiv, Any, (Any, UInt), ci, 0x0)::CodeInstance !== ci end return Core.svec(codeinfos, cis) diff --git a/src/gf.c b/src/gf.c index db6c10efc958a..1eb306d3ac599 100644 --- a/src/gf.c +++ b/src/gf.c @@ -631,7 +631,7 @@ static int jl_codeinst_edges_sub(jl_code_instance_t *ci, size_t min_world2, size size_t max_world = jl_atomic_load_relaxed(&ci->max_world); jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); // n.b.: edges matching is implied sufficiently by the matching worlds for min_world, but not max_world == ~0 (which couldn't compute that) - return min_world >= min_world2 && max_world <= max_world2 && edges && jl_egal((jl_value_t*)edges, (jl_value_t*)edges2); + return min_world >= min_world2 && max_world <= max_world2 && edges; // TODO: && jl_egal((jl_value_t*)edges, (jl_value_t*)edges2); } // return whether the codeinst can be substituted in place of ci for an invoke target in target_world @@ -3811,51 +3811,6 @@ static int need_copy_to_mi_cache(jl_method_instance_t *mi, jl_method_instance_t !jl_egal((jl_value_t*)mi->sparam_vals, (jl_value_t*)mi2->sparam_vals); } -// Link callee to cached so that invalidation of cached propagates to callee. -// When a CodeInstance swap occurs (callee replaced by cached during compilation), -// we must ensure callee gets invalidated whenever cached is, since any caller -// holding callee by pointer identity in its edges won't otherwise be reached. -// Return 1 if cached should replace callee (cached covers a wider world range), -// or 0 if cached object is no longer current (world counter invalidated it concurrently). -// Sets up edges and backedges so that invalidation of cached propagates to callee. -//JL_DLLEXPORT int jl_link_ci_equiv(jl_code_instance_t *callee, jl_code_instance_t *cached) -//{ -// if (callee == cached) -// return 1; -// if (jl_atomic_load_relaxed(&callee->max_world) != ~(size_t)0) -// return 1; -// jl_method_instance_t *mi = jl_get_ci_mi(cached); -// if (!jl_is_method(mi->def.method)) -// return 1; -// // Take a lock here to simplify the ordering needed here by ensuring worlds and edges don't change during these steps -// JL_LOCK(&world_counter_lock); -// int add_edge = jl_atomic_load_relaxed(&cached->max_world) == ~(size_t)0; -// if (add_edge) { -// // Append cached to callee->edges so _invalidate_backedges can find it by identity -// int already_linked = 0; -// jl_svec_t *old_edges = jl_atomic_load_relaxed(&callee->edges); -// size_t old_n = jl_svec_len(old_edges); -// for (size_t i = 0; i < old_n; i++) { -// if (jl_svecref(old_edges, i) == (jl_value_t *)cached) { -// already_linked = 1; -// break; -// } -// } -// if (!already_linked) { -// jl_svec_t *new_edges = jl_alloc_svec(old_n + 1); -// memcpy(jl_svec_data(new_edges), jl_svec_data(old_edges), old_n * sizeof(jl_value_t *)); -// jl_svecset(new_edges, old_n, (jl_value_t *)cached); -// // an invoke-type edge would be more accurate here, but this is conservatively correct -// jl_gc_write_atomic(callee, callee->edges, jl_svec_t, new_edges, release); -// // register callee in cached's MI backedges so invalidation of cached propagates to callee -// jl_method_instance_add_backedge(mi, NULL, callee); -// } -// } -// JL_UNLOCK(&world_counter_lock); -// return add_edge; -//} - - static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_code_instance_t *codeinst2) JL_CANSAFEPOINT { size_t current_world = jl_get_world_counter(); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 865d60f181c19..c9c453f9feff5 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2301,7 +2301,6 @@ renameLinkGraphSymbol(jitlink::LinkGraph &G, jitlink::Symbol &Sym, bool JuliaOJIT::linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferRef ObjBuf, jitlink::LinkGraph &G, std::unique_ptr Info) - JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { std::unique_lock Lock{LinkerMutex}; From f221cc6bd95a43bd012ea8a201a65b130718f1c5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 15 Jul 2026 20:36:11 +0000 Subject: [PATCH 6/6] fixup! defer real fix --- src/gf.c | 4 +++- src/jitlayers.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/gf.c b/src/gf.c index 1eb306d3ac599..4dd947d5543ea 100644 --- a/src/gf.c +++ b/src/gf.c @@ -625,7 +625,7 @@ JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, } // return whether the ci has more restrictions than the other arguments (more edges and narrower worlds) -static int jl_codeinst_edges_sub(jl_code_instance_t *ci, size_t min_world2, size_t max_world2, jl_svec_t *edges2) +static int jl_codeinst_edges_sub(jl_code_instance_t *ci, size_t min_world2, size_t max_world2, jl_svec_t *edges2) JL_NOTSAFEPOINT { size_t min_world = jl_atomic_load_relaxed(&ci->min_world); size_t max_world = jl_atomic_load_relaxed(&ci->max_world); @@ -3819,12 +3819,14 @@ static jl_code_instance_t *copy_to_mi_cache(jl_method_instance_t *mi JL_PROPAGAT // it so that invalidation of codeinst2 also invalidates codeinst jl_method_t *m = mi->def.method; jl_svec_t *copy_edge = jl_is_method(m) ? jl_svec2(m->sig, codeinst2) : jl_emptysvec; + JL_GC_PUSH1(©_edge); jl_code_instance_t *codeinst = jl_get_method_uninferred( mi, codeinst2->rettype, jl_atomic_load_relaxed(&codeinst2->min_world), max_world2 < current_world ? max_world2 : current_world, jl_atomic_load_relaxed(&codeinst2->debuginfo), copy_edge); + JL_GC_POP(); if (jl_atomic_load_relaxed(&codeinst->invoke) == NULL) { if (max_world2 == ~(size_t)0) { JL_LOCK(&world_counter_lock); diff --git a/src/jitlayers.h b/src/jitlayers.h index cb9e86b99909c..c22304ad144da 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -890,7 +890,7 @@ class JuliaOJIT { // false after calling MR.failMaterialization(). bool linkOutput(orc::MaterializationResponsibility &MR, MemoryBufferRef ObjBuf, jitlink::LinkGraph &G, - std::unique_ptr Info) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER; + std::unique_ptr Info) JL_CANSAFEPOINT_ENTER_LEAVE; // Return a symbol that should be linked to the call target. The origin of // this symbol depends on the code instance: