From 2eb6a63cc9883bcd90569bdb72b7219b58247677 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 15 Jul 2026 17:43:01 -0400 Subject: [PATCH 1/5] typemap: add jl_typemap_list_t record lists Add shared machinery for caches that key a TypeMap exactly on a signature type and hold several records per entry: an intrusive, append-only list chained through an atomic next field embedded in each record, walked lock-free and mutated under the owning cache's writelock. Keys stay packed inline in the records; a match callback reads them directly. Co-Authored-By: Claude Fable 5 --- src/julia_internal.h | 18 ++++++++++ src/typemap.c | 85 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/src/julia_internal.h b/src/julia_internal.h index 7c2881a2398f3..baf0f03c077b5 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1902,6 +1902,24 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( struct jl_typemap_assoc *search, int8_t offs, uint8_t subtype) JL_CANSAFEPOINT; +// jl_typemap_list_t: an intrusive, append-only record list held as the value of a TypeMap +// entry keyed exactly on a signature type, chained through an atomic `next` field embedded +// in each record at `next_offset` (the ABI-adapter and dispatch-trampoline caches). Keys +// stay packed inline in the records; `match` reads them directly. Reads are lock-free; +// mutation requires the owning cache's writelock. +typedef jl_value_t jl_typemap_list_t; +typedef int (*jl_typemap_list_match_t)(jl_value_t *rec, void *key); +JL_DLLEXPORT jl_value_t *jl_typemap_list_find(jl_typemap_list_t *head JL_PROPAGATES_ROOT, size_t next_offset, + jl_typemap_list_match_t match JL_CANSAFEPOINT, void *key) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_typemap_entry_t *jl_typemap_list_entry(_Atomic(jl_typemap_t*) *cache JL_PROPAGATES_ROOT, + jl_value_t *sigt) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_typemap_list_lookup(_Atomic(jl_typemap_t*) *cache JL_PROPAGATES_ROOT, jl_value_t *sigt, + size_t next_offset, jl_typemap_list_match_t match JL_CANSAFEPOINT, void *key) JL_CANSAFEPOINT; +JL_DLLEXPORT void jl_typemap_list_append(jl_typemap_list_t *head, jl_value_t *rec, + size_t next_offset) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_typemap_list_insert(_Atomic(jl_typemap_t*) *cache, jl_value_t *owner, + jl_value_t *sigt, jl_value_t *rec, size_t next_offset) JL_CANSAFEPOINT; + jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world) JL_CANSAFEPOINT; jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *mn, jl_value_t *arg1, jl_value_t **args, size_t n, size_t world) JL_CANSAFEPOINT; STATIC_INLINE jl_typemap_entry_t *jl_typemap_assoc_exact( diff --git a/src/typemap.c b/src/typemap.c index 90d93795c3640..a89c46cb8df74 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -424,6 +424,91 @@ int jl_typemap_visitor(jl_typemap_t *cache, jl_typemap_visitor_fptr fptr, void * } } +// ---- jl_typemap_list_t: sigt-keyed record lists ---- +// Shared machinery for caches that key a TypeMap *exactly* on a signature type and hold +// several records per entry (the ABI-adapter and dispatch-trampoline caches): the entry's +// value is a jl_typemap_list_t, an intrusive, append-only list chained through an atomic +// `next` field embedded in each record at `next_offset`. Keys stay packed inline in the +// records; the `match` callback reads them directly. Readers walk lock-free: entries are +// created valid for all worlds, the list head in `entry->func.value` never moves, and +// appends publish with release. Mutation requires the owning cache's writelock. + +static inline _Atomic(jl_value_t*) *typemap_list_next(jl_value_t *rec, size_t next_offset) JL_NOTSAFEPOINT +{ + return (_Atomic(jl_value_t*)*)((char*)rec + next_offset); +} + +// Walk a record list (acquire) for a record satisfying `match`; NULL if none. +JL_DLLEXPORT jl_value_t *jl_typemap_list_find(jl_typemap_list_t *head JL_PROPAGATES_ROOT, size_t next_offset, + jl_typemap_list_match_t match, void *key) JL_CANSAFEPOINT +{ + for (jl_value_t *e = head; e != NULL; e = jl_atomic_load_acquire(typemap_list_next(e, next_offset))) { + JL_GC_PROMISE_ROOTED(e); // rooted by the (append-only) cache TypeMap + if (match(e, key)) + return e; + } + return NULL; +} + +// The TypeMap entry holding `sigt`'s record list (exact match, like jl_methtable_lookup); +// NULL if absent. Safe with or without the cache writelock. +JL_DLLEXPORT jl_typemap_entry_t *jl_typemap_list_entry(_Atomic(jl_typemap_t*) *cache JL_PROPAGATES_ROOT, + jl_value_t *sigt) JL_CANSAFEPOINT +{ + jl_typemap_t *map = jl_atomic_load_relaxed(cache); + if ((jl_value_t*)map == jl_nothing) + return NULL; + struct jl_typemap_assoc search = { sigt, jl_atomic_load_acquire(&jl_world_counter), NULL }; + return jl_typemap_assoc_by_type(map, &search, /*offs*/0, /*subtype*/0); +} + +// Lock-free lookup of the record matching `key` in `sigt`'s list; NULL if absent. +JL_DLLEXPORT jl_value_t *jl_typemap_list_lookup(_Atomic(jl_typemap_t*) *cache JL_PROPAGATES_ROOT, jl_value_t *sigt, + size_t next_offset, jl_typemap_list_match_t match, void *key) JL_CANSAFEPOINT +{ + jl_typemap_entry_t *te = jl_typemap_list_entry(cache, sigt); + if (te == NULL) + return NULL; + jl_typemap_list_t *head = jl_atomic_load_acquire((_Atomic(jl_value_t*)*)&te->func.value); + return jl_typemap_list_find(head, next_offset, match, key); +} + +// Append `rec` at the tail of the list headed by `head` (the head never moves, so readers +// stay lock-free). Caller holds the cache writelock and keeps `rec` rooted; `rec`'s next +// field must be NULL. +JL_DLLEXPORT void jl_typemap_list_append(jl_typemap_list_t *head, jl_value_t *rec, + size_t next_offset) JL_NOTSAFEPOINT +{ + jl_value_t *tail = head; + for (;;) { + jl_value_t *n = jl_atomic_load_relaxed(typemap_list_next(tail, next_offset)); + if (n == NULL) + break; + tail = n; + } + jl_atomic_store_release(typemap_list_next(tail, next_offset), rec); + jl_gc_wb(tail, rec); +} + +// Insert `rec` into `sigt`'s list: the first record becomes a new all-worlds TypeMap entry, +// later ones are appended at the tail. Caller holds the cache writelock, has confirmed +// `rec`'s key absent, and keeps `sigt`/`rec` rooted; `rec`'s next field must be NULL. +JL_DLLEXPORT void jl_typemap_list_insert(_Atomic(jl_typemap_t*) *cache, jl_value_t *owner, + jl_value_t *sigt, jl_value_t *rec, size_t next_offset) JL_CANSAFEPOINT +{ + jl_typemap_entry_t *te = jl_typemap_list_entry(cache, sigt); + if (te == NULL) { + jl_typemap_entry_t *newentry = jl_typemap_alloc((jl_tupletype_t*)sigt, NULL, + jl_emptysvec, rec, 1, ~(size_t)0); + JL_GC_PUSH1(&newentry); + jl_typemap_insert(cache, owner, newentry, /*offs*/0); + JL_GC_POP(); + return; + } + jl_typemap_list_t *head = jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)&te->func.value); + jl_typemap_list_append(head, rec, next_offset); +} + static unsigned jl_supertype_height(jl_datatype_t *dt) JL_NOTSAFEPOINT { unsigned height = 1; From 3e290ba43e231c98e69accd87c3b3b38431a56f0 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 1 Jul 2026 13:35:39 -0400 Subject: [PATCH 2/5] codegen: add interned ABIAdapter cache and serialization Introduce the ABIAdapter datatype and a process-global adapter cache (`Core.abi_adapters`, an ABIAdapterCache with its own writelock), interning one JIT'd ABI-adapter thunk per distinct (sigt, rt, ci, specsig, is_opaque_closure, nargs) rather than one per call site. Adapters serialize as first-class image code: records are fvar-wired (jl_get_adapter_id), reset their bucket `next`, and are re-interned on load (jl_reintern_abi_adapter). Because an adapter has no compiled-code edge of its own, it is reported to the serializer via `ext_foreign_code` (renamed from ext_foreign_cis) so independent adapters -- ones not owned by any trampoline -- are rooted through the pre-dump GC and serialized/reinterned in both incremental and full/AOT images. The codegen-free fallback (jl_jit_abi_converter_fallback) is cache-aware. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Claude Fable 5 --- base/methodshow.jl | 2 + src/Makefile | 2 +- src/abi_adapter.c | 301 ++++++++++++++++++++++++++++++++++++++ src/aotcompile.cpp | 68 ++++++++- src/builtins.c | 3 + src/codegen-stubs.c | 30 +++- src/datatype.c | 13 ++ src/jitlayers.cpp | 107 +++++++++----- src/jitlayers.h | 4 + src/jl_exported_data.inc | 3 + src/jl_exported_funcs.inc | 1 + src/jltypes.c | 38 +++++ src/julia.h | 68 +++++++++ src/julia_internal.h | 29 +++- src/runtime_ccall.c | 4 +- src/staticdata.c | 130 ++++++++++++++-- src/typemap.c | 18 +++ 17 files changed, 760 insertions(+), 61 deletions(-) create mode 100644 src/abi_adapter.c diff --git a/base/methodshow.jl b/base/methodshow.jl index cb85d8a29a32d..8dd53ef498046 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -387,6 +387,8 @@ end show(io::IO, ms::MethodList) = show_method_table(io, ms) show(io::IO, ::MIME"text/plain", ms::MethodList) = show_method_table(io, ms) show(io::IO, mt::Core.MethodTable) = print(io, mt.module, ".", mt.name, " is a Core.MethodTable with ", length(mt), " methods.") +show(io::IO, c::Core.ABIAdapterCache) = print(io, "Core.ABIAdapterCache with ", + ccall(:jl_typemap_count, Csize_t, (Any,), getfield(c, :cache)), " entries.") function inbase(m::Module) if m == Base diff --git a/src/Makefile b/src/Makefile index f97dab7d0e430..89e5172580633 100644 --- a/src/Makefile +++ b/src/Makefile @@ -67,7 +67,7 @@ SRCS := \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ threading scheduler stackwalk null_sysimage \ method jlapi signal-handling safepoint timing subtype rtutils \ - crc32c APInt processor ircode opaque_closure codegen-stubs coverage runtime_ccall engine \ + crc32c APInt processor ircode opaque_closure abi_adapter codegen-stubs coverage runtime_ccall engine \ $(GC_SRCS) RT_LLVMLINK := diff --git a/src/abi_adapter.c b/src/abi_adapter.c new file mode 100644 index 0000000000000..5ff171067d0bd --- /dev/null +++ b/src/abi_adapter.c @@ -0,0 +1,301 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "julia.h" +#include "julia_internal.h" + +// Process-global cache backing `jl_abi_adapters` (the `Core.abi_adapters` singleton). +// See the section comment below for the cache / bucket structure. + +// ---- ABI adapter cache (`jl_abi_adapters->cache`) ---- +// A TypeMap keyed on the adapter `sigt` alone (its +// slot 0 is the function type, never `Union{}`). Each TypeMap entry holds a *bucket* of +// jl_abi_adapter_t records that share one `sigt` but differ in (rt, ci, specsig, +// is_opaque_closure, nargs). Because an adapter `sigt` may be shared by many distinct target +// `ci`, the bucket is adaptive, mirroring the TypeMap's own single/list/cache trichotomy: +// tier 1-2: `entry->func.value` is a `.next`-chained list of records (kept while +// count <= MAX_ADAPTER_LIST_COUNT); appended at the tail so the head +// never moves and readers walk it lock-free; +// tier 3: promoted to a `jl_eqtable` (Memory) keyed on the record's `ci` +// (NULL -> jl_nothing), whose value is a short `.next` chain of the +// same-ci variants. +// `entry->func.value` is read with acquire / written with release (it is swapped on +// promotion), so lock-free readers see a complete old-or-new representation. A reader +// mid-walk during promotion may spuriously miss and must recover via a writelock-held +// recheck (jl_lookup_abi_converter). The record lists are jl_typemap_list_t (typemap.c); the +// eqtable tiering is local to this cache. The JIT half (LLVM emit + record allocation) +// lives in jitlayers.cpp. + +#define MAX_ADAPTER_LIST_COUNT 6 // mirror MAX_METHLIST_COUNT: promote chain -> eqtable above this + +// Record key within a `sigt` bucket; the fields live packed inline on the record itself. +typedef struct { + jl_value_t *rt; + jl_code_instance_t *ci; + int specsig; + int is_opaque_closure; + size_t nargs; +} abi_adapter_key_t; + +// `rt` is compared by *type equality* (jl_types_equal), mirroring the TypeMap's `sigt` match; +// jl_egal would split type-equal-but-not-egal return types into duplicate records. +static int abi_adapter_match(jl_value_t *rec, void *keyv) JL_CANSAFEPOINT +{ + jl_abi_adapter_t *e = (jl_abi_adapter_t*)rec; + abi_adapter_key_t *k = (abi_adapter_key_t*)keyv; + return e->ci == k->ci + && (int)e->specsig == (k->specsig ? 1 : 0) + && (int)e->is_opaque_closure == (k->is_opaque_closure ? 1 : 0) + && e->nargs == k->nargs + && (e->rt == k->rt || jl_types_equal(e->rt, k->rt)); // rt lives in the bucket (key is sigt alone) +} + +// eqtable key for a record's `ci` (eqtable can't key NULL -> jl_nothing sentinel). +static jl_value_t *abi_adapter_cikey(jl_code_instance_t *ci) JL_NOTSAFEPOINT +{ + return ci ? (jl_value_t*)ci : jl_nothing; +} + +// Find a record in `te`'s bucket. `repr` (acquire) is either a record list (tiers 1-2) or a +// `ci`-keyed eqtable of record lists (tier 3); both reduce to a short list walk. +static jl_abi_adapter_t *abi_adapter_bucket_find(jl_typemap_entry_t *te, abi_adapter_key_t *key) JL_CANSAFEPOINT +{ + jl_value_t *repr = jl_atomic_load_acquire((_Atomic(jl_value_t*)*)&te->func.value); + if (repr == NULL) + return NULL; + jl_typemap_list_t *head = repr; + if (jl_is_genericmemory(repr)) + head = jl_eqtable_get((jl_genericmemory_t*)repr, abi_adapter_cikey(key->ci), NULL); + return (jl_abi_adapter_t*)jl_typemap_list_find(head, offsetof(jl_abi_adapter_t, next), + abi_adapter_match, key); +} + +// Whether a target entry point of kind `api` (belonging to `mi`, returning `rettype`) +// already satisfies the caller ABI `from_abi`, so no wrapping adapter is needed. The AOT path +// passes the api of the Function it emitted (decls.invoke_api); the runtime derives it from +// the published entry points (abi_adapter_resolve_target). +JL_DLLEXPORT int jl_abi_matches_invoke_api(jl_abi_t from_abi, jl_invoke_api_t api, + jl_method_instance_t *mi, jl_value_t *rettype) JL_CANSAFEPOINT +{ + if (api == JL_INVOKE_ARGS) + return !from_abi.specsig && jl_subtype(rettype, from_abi.rt); + if (api == JL_INVOKE_SPECSIG) + return from_abi.specsig && jl_egal(mi->specTypes, from_abi.sigt) && jl_egal(rettype, from_abi.rt); + return 0; +} + +// Resolve whether (`from_abi`, `codeinst`) needs a wrapping adapter at all. If the target's +// own compiled entry point already satisfies the requested ABI, return that specptr directly +// (no adapter). Otherwise return NULL and fill *target / *target_specsig / *invoke describing +// the target the adapter must bridge to. Pure metadata resolution: no codegen and no +// compilation, so it is also usable in builds without codegen. +static void *abi_adapter_resolve_target(jl_abi_t from_abi, jl_code_instance_t *codeinst, + void **target, int *target_specsig, jl_callptr_t *invoke) JL_CANSAFEPOINT +{ + *target = NULL; + *target_specsig = 0; + *invoke = NULL; + if (codeinst == NULL) + return NULL; + uint8_t specsigflags; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + void *specptr = NULL; + jl_callptr_t inv = NULL; + // `waitcompile` must be 0 to keep this callable without codegen. A target still + // mid-compilation reads back as `inv == NULL`: no shortcut. + jl_read_codeinst_invoke(codeinst, &specsigflags, &inv, &specptr, /* waitcompile */ 0); + *invoke = inv; + if (inv == NULL) + return NULL; + if (inv == jl_fptr_const_return_addr) { + // const-return has no specptr to shortcut to: an adapter is always required. + return NULL; + } + else if (inv == jl_fptr_args_addr) { + assert(specptr != NULL); + if (jl_abi_matches_invoke_api(from_abi, JL_INVOKE_ARGS, mi, codeinst->rettype)) + return specptr; // no adapter required + *target = specptr; + *target_specsig = 0; + } + else if (specsigflags & JL_CI_FLAGS_SPECPTR_SPECIALIZED) { + assert(specptr != NULL); + if (jl_abi_matches_invoke_api(from_abi, JL_INVOKE_SPECSIG, mi, codeinst->rettype)) + return specptr; // no adapter required + *target = specptr; + *target_specsig = 1; + } + return NULL; +} + +// The target's own compiled entry point when it already satisfies `from_abi` (so no adapter +// is needed), else NULL. Pure metadata read; usable without codegen. +JL_DLLEXPORT void *jl_abi_matching_specptr(jl_abi_t from_abi, jl_code_instance_t *codeinst) JL_CANSAFEPOINT +{ + void *target; + int target_specsig; + jl_callptr_t invoke; + return abi_adapter_resolve_target(from_abi, codeinst, &target, &target_specsig, &invoke); +} + +// Lock-free lookup of the adapter record for (sigt, rt, ci, specsig, is_opaque_closure, +// nargs). Returns the record or NULL. Safe to call with or without the writelock held. +JL_DLLEXPORT jl_abi_adapter_t *jl_lookup_abi_adapter(jl_value_t *sigt, jl_value_t *rt, + jl_code_instance_t *ci, int specsig, int is_opaque_closure, size_t nargs) JL_CANSAFEPOINT +{ + // Key on `sigt` alone (its slot 0 is the function type, never Union{}); the remaining + // fields are disambiguated in the bucket (abi_adapter_match). + abi_adapter_key_t key = { rt, ci, specsig, is_opaque_closure, nargs }; + jl_typemap_entry_t *te = jl_typemap_list_entry(&jl_abi_adapters->cache, sigt); + if (te == NULL) + return NULL; + return abi_adapter_bucket_find(te, &key); +} + +// Find a callable adapter fptr for (from_abi, codeinst) without JITing: the target's own +// specptr when its compiled ABI already matches, else a cached record. A lock-free +// miss is re-checked under the writelock (a concurrent bucket promotion can cause a spurious +// miss; see the section comment above). Returns NULL when no adapter exists: the JIT path +// then compiles one, and a codegen-free caller can re-query with codeinst == NULL for the +// signature's dynamic-dispatch adapter. All out parameters are optional: `invokee` receives +// the Union{CodeInstance, ABIAdapter} behind the returned fptr; the rest receive the +// resolve-target results for a caller that proceeds to JIT. +JL_DLLEXPORT void *jl_lookup_abi_converter(jl_abi_t from_abi, jl_code_instance_t *codeinst, + void **target, int *target_specsig, jl_callptr_t *invoke, jl_value_t **invokee) JL_CANSAFEPOINT +{ + void *tgt = NULL; + int ts = 0; + jl_callptr_t inv = NULL; + if (invokee) + *invokee = NULL; + void *shortcut = abi_adapter_resolve_target(from_abi, codeinst, &tgt, &ts, &inv); + if (target) + *target = tgt; + if (target_specsig) + *target_specsig = ts; + if (invoke) + *invoke = inv; + if (shortcut != NULL) { + if (invokee) + *invokee = (jl_value_t*)codeinst; // bare CI: no adapter record required + return shortcut; + } + jl_abi_adapter_t *e = jl_lookup_abi_adapter(from_abi.sigt, from_abi.rt, codeinst, + from_abi.specsig, from_abi.is_opaque_closure, from_abi.nargs); + if (e == NULL || e->fptr == NULL) { + JL_LOCK(&jl_abi_adapters->writelock); + e = jl_lookup_abi_adapter(from_abi.sigt, from_abi.rt, codeinst, + from_abi.specsig, from_abi.is_opaque_closure, from_abi.nargs); + JL_UNLOCK(&jl_abi_adapters->writelock); + } + // A record without a compiled thunk (fptr == NULL) cannot be served; report a miss so the + // caller JITs one (jl_get_abi_adapter fills such a record in place) or errors cleanly. + if (e == NULL || e->fptr == NULL) + return NULL; + if (invokee) + *invokee = (jl_value_t*)e; + return e->fptr; +} + +// Insert `e` (with `e->next` published via release) into `tbl`'s `ci`-bucket chain by +// prepending. Returns the (possibly reallocated) table. +static jl_genericmemory_t *abi_adapter_eqtable_prepend(jl_genericmemory_t *tbl, jl_abi_adapter_t *e) JL_CANSAFEPOINT +{ + jl_value_t *cik = abi_adapter_cikey(e->ci); + jl_abi_adapter_t *head = (jl_abi_adapter_t*)jl_eqtable_get(tbl, cik, NULL); + // `e` may be an old-gen record (promotion regroups existing chain records; + // get-or-insert adds in-image records), so the new e -> head edge needs the write barrier. + jl_gc_write_atomic(e, e->next, jl_abi_adapter_t, head, release); + int inserted = 0; + return jl_eqtable_put(tbl, cik, (jl_value_t*)e, &inserted); +} + +// Insert `entry` into the adapters cache. Caller holds the writelock and must have +// confirmed (under the lock) that the key is absent. +static void abi_adapter_insert(jl_abi_adapter_t *entry) JL_CANSAFEPOINT +{ + jl_genericmemory_t *tbl = NULL; + jl_abi_adapter_t *e = NULL, *nxt = NULL; + JL_GC_PUSH4(&entry, &tbl, &e, &nxt); + // Key on the adapter sig alone; rt is in the bucket (see jl_lookup_abi_adapter). + jl_value_t *key = entry->sigt; // rooted via `entry` + jl_typemap_entry_t *te = jl_typemap_list_entry(&jl_abi_adapters->cache, key); + if (te == NULL) { + jl_typemap_list_insert(&jl_abi_adapters->cache, (jl_value_t*)jl_abi_adapters, key, + (jl_value_t*)entry, offsetof(jl_abi_adapter_t, next)); + JL_GC_POP(); + return; + } + jl_value_t *repr = jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)&te->func.value); + if (jl_is_genericmemory(repr)) { + // tier 3: prepend into the ci-bucket; publish a grown table if it reallocated. + tbl = abi_adapter_eqtable_prepend((jl_genericmemory_t*)repr, entry); + if ((jl_value_t*)tbl != repr) { + jl_atomic_store_release((_Atomic(jl_value_t*)*)&te->func.value, (jl_value_t*)tbl); + jl_gc_wb(te, tbl); + } + JL_GC_POP(); + return; + } + // tiers 1-2: a record list. Count it to decide between appending and promoting. + jl_abi_adapter_t *head = (jl_abi_adapter_t*)repr; + size_t count = 1; + for (jl_abi_adapter_t *n = head; (n = jl_atomic_load_relaxed(&n->next)) != NULL; ) + count++; + if (count < MAX_ADAPTER_LIST_COUNT) { + jl_typemap_list_append((jl_value_t*)head, (jl_value_t*)entry, + offsetof(jl_abi_adapter_t, next)); + JL_GC_POP(); + return; + } + // tier 2 -> 3: promote the chain to a ci-keyed eqtable, then publish it. Regrouping + // relinks `.next` (concurrent old-chain readers may miss -> locked recheck recovers). + tbl = jl_alloc_memory_any(16); + e = head; + while (e != NULL) { + nxt = jl_atomic_load_relaxed(&e->next); + tbl = abi_adapter_eqtable_prepend(tbl, e); + e = nxt; + } + tbl = abi_adapter_eqtable_prepend(tbl, entry); + jl_atomic_store_release((_Atomic(jl_value_t*)*)&te->func.value, (jl_value_t*)tbl); + jl_gc_wb(te, tbl); + JL_GC_POP(); +} + +// Allocate an ABI-adapter cache record. Caller keeps the key fields (`sigt`/`rt`/`ci`) +// rooted. `next` starts as the chain tail. +JL_DLLEXPORT jl_abi_adapter_t *jl_new_abi_adapter(jl_value_t *sigt, jl_value_t *rt, + jl_code_instance_t *ci, int specsig, int is_opaque_closure, size_t nargs, void *fptr) JL_CANSAFEPOINT +{ + jl_task_t *ct = jl_current_task; + jl_abi_adapter_t *e = (jl_abi_adapter_t*)jl_gc_alloc(ct->ptls, sizeof(jl_abi_adapter_t), jl_abi_adapter_type); + e->sigt = sigt; + e->rt = rt; + e->specsig = specsig ? 1 : 0; + e->is_opaque_closure = is_opaque_closure ? 1 : 0; + e->nargs = nargs; + e->ci = ci; + e->fptr = fptr; + jl_atomic_store_relaxed(&e->next, (jl_abi_adapter_t*)NULL); + return e; +} + +// Insert `e` into the running cache if its key is absent and return the canonical record; +// keep-first, like jl_specializations_get_or_insert (a losing `e` is left standalone). The +// JIT path inserts freshly-built records here (under its already-held writelock; the lock is +// reentrant), and the load fixup re-inserts image-restored records, whose `fptr` must already +// be wired (jl_update_all_fptrs). +JL_DLLEXPORT jl_abi_adapter_t *jl_insert_abi_adapter(jl_abi_adapter_t *e) JL_CANSAFEPOINT +{ + jl_abi_adapter_t *canonical = NULL; + JL_GC_PUSH2(&e, &canonical); + JL_LOCK(&jl_abi_adapters->writelock); + canonical = jl_lookup_abi_adapter(e->sigt, e->rt, e->ci, e->specsig, e->is_opaque_closure, e->nargs); + if (canonical == NULL) { + abi_adapter_insert(e); + canonical = e; + } + JL_UNLOCK(&jl_abi_adapters->writelock); + JL_GC_POP(); + return canonical; +} diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 5de83829aaf1a..9c1da209d0068 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -176,6 +176,14 @@ typedef struct { // were presented in `codeinfos`; consumed by staticdata.c to rewrite each // MethodInstance's `cache` field into a `next`-linked list SmallVector jl_ci_order; + // ABI-adapter records emitted by the AOT @cfunction/@ccallable path. They are + // registered as fvars lazily, only when the + // serializer reaches an ABIAdapter record and calls jl_get_adapter_id for it, so the image + // compiles adapters only for records that are actually serialized. Each record gets its own + // fvar slot (jl_get_adapter_id), so `fptr_record` stays 1:1 (fvar id -> one record) even when + // records share an adapter Function -- exactly as CodeInstances get a fvar slot each. + std::map pending_adapters; // emitted, maybe-unregistered + std::map adapter_fvar_map; // record -> fvar index } jl_native_code_desc_t; extern "C" JL_DLLEXPORT_CODEGEN @@ -192,6 +200,40 @@ void jl_get_function_id_impl(void *native_code, jl_code_instance_t *codeinst, } } +// Look up (registering on first use) the fvar index of the compiled adapter behind ABIAdapter +// record `rec`. The serializer calls this once per serialized record, which drives the lazy +// registration: only serialized records' adapters become fvars and survive into the image. Sets +// `*adapter_idx` to the 1-based index, or leaves it untouched (0) if no adapter was emitted. +extern "C" JL_DLLEXPORT_CODEGEN +void jl_get_adapter_id_impl(void *native_code, jl_abi_adapter_t *rec, int32_t *adapter_idx) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (!data) + return; + auto it = data->adapter_fvar_map.find(rec); + if (it != data->adapter_fvar_map.end()) { + *adapter_idx = it->second; + return; + } + auto pit = data->pending_adapters.find(rec); + if (pit == data->pending_adapters.end()) + return; // no adapter was emitted for this record + Function *F = pit->second; + // Allocate a fresh fvar slot for this record -- mirroring the per-CodeInstance fvar allocation + // in jl_emit_native_to_output, which pushes an fvar per CI unconditionally. `fptr_record` is + // keyed by fvar id and stores a *single* record location per slot; the load-time restore + // (jl_update_all_fptrs) walks fvar ids and writes each slot's one object. A per-record slot + // therefore keeps that mapping 1:1 by construction, even when two records happen to share an + // adapter Function (the fvar table tolerates the same llvm::Function at multiple indices -- + // CodeInstances that share code via get_ci_equiv already do exactly that). Deduping the slot + // by Function instead would funnel two records into one `fptr_record` slot, dropping one + // record's `fptr` on load (a consumer pointing at it would then call a null pointer). + data->jl_sysimg_fvars.push_back(F); + uint32_t idx = data->jl_sysimg_fvars.size(); + data->adapter_fvar_map[rec] = idx; + *adapter_idx = idx; +} + extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_cis_impl(void *native_code, size_t *num_elements, jl_code_instance_t **data) { @@ -523,6 +565,11 @@ static void aot_optimize_roots(jl_codegen_output_t &out, egal_set &method_roots) } } +// Emit a fresh, uniquely-named adapter Function. Like each CodeInstance, every ABIAdapter record +// gets its own distinct Function, so the fvar table has no duplicates (get_fvars_gvars) and +// `fptr_record` stays 1:1 (fvar id -> one record). Two @cfunctions resolving to equivalent adapters +// emit their own thunks -- as CodeInstances do for equivalent code -- rather than sharing one +// (which would break that 1:1 mapping); adapters are not deduplicated at emission. static Function *aot_abi_converter(jl_codegen_output_t &out, jl_abi_t from_abi, jl_code_instance_t *codeinst, Function *func, Function *specfunc, bool target_specsig) JL_CANSAFEPOINT { std::string gf_thunk_name; @@ -630,7 +677,7 @@ static bool canPartition(const Function &F) // `external_linkage` create linkages between pkgimages. extern "C" JL_DLLEXPORT_CODEGEN void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int external_linkage, size_t world, - jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_cis) + jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_code) { JL_TIMING(INFERENCE, INFERENCE); auto ct = jl_current_task; @@ -667,7 +714,7 @@ void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int ex fargs[5] = mod_array ? (jl_value_t*)mod_array : jl_nothing; // mod_array (or nothing) fargs[6] = jl_box_bool(all); fargs[7] = (jl_value_t*)module_init_order; // module_init_order - fargs[8] = ext_foreign_cis ? (jl_value_t*)ext_foreign_cis : jl_nothing; // ext_foreign_cis (or nothing) + fargs[8] = ext_foreign_code ? (jl_value_t*)ext_foreign_code : jl_nothing; // ext_foreign_code (or nothing) size_t last_age = ct->world_age; ct->world_age = jl_typeinf_world; fargs[0] = jl_apply(fargs, 9); @@ -683,6 +730,17 @@ void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int ex JL_TYPECHK(jl_create_native, array_any, codeinfos); JL_TYPECHK(jl_create_native, array_any, ci_order); auto data = (jl_native_code_desc_t *)jl_emit_native((jl_array_t*)codeinfos, (jl_array_t*)ci_order, llvmmod, NULL, external_linkage ? 1 : 0); + // Report each compiled ABI adapter as extra image code (via `ext_foreign_code`, like an + // external CodeInstance), which roots it through the pre-dump GC and gets it serialized + + // reinterned on load even when nothing else owns it. The records are kept alive across + // emission in `data->out->temporary_roots`; re-root that array so the pushes are GC-safe. + if (ext_foreign_code && data->out && data->out->temporary_roots) { + jl_array_t *troots = data->out->temporary_roots; + JL_GC_PUSH1(&troots); + for (auto &kv : data->pending_adapters) + jl_array_ptr_1d_push(ext_foreign_code, (jl_value_t*)kv.first); + JL_GC_POP(); + } JL_GC_POP(); // move everything inside, now that we've merged everything @@ -995,6 +1053,12 @@ static void jl_emit_native_to_output(jl_native_code_desc_t *data, jl_array_t *co } data->jl_fvar_map[ci] = {invoke_id, specptr_id}; } + // Record the emitted ABIAdapter records as *pending* (registered as fvars lazily by + // jl_get_adapter_id when the serializer reaches each record). Reporting them to + // `ext_foreign_code` (so they're rooted + serialized even when nothing else owns them) is + // done by the caller (jl_create_native_impl), which has that array in scope. + for (auto &[rec, F] : out.abi_adapter_records) + data->pending_adapters[rec] = F; } // also be used by extern consumers like GPUCompiler.jl to obtain a module containing diff --git a/src/builtins.c b/src/builtins.c index 6af2dc5ebeae5..b2cec38bea5a5 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2718,6 +2718,9 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("CodeInstance", (jl_value_t*)jl_code_instance_type); add_builtin("TypeMapEntry", (jl_value_t*)jl_typemap_entry_type); add_builtin("TypeMapLevel", (jl_value_t*)jl_typemap_level_type); + add_builtin("ABIAdapter", (jl_value_t*)jl_abi_adapter_type); + add_builtin("ABIAdapterCache", (jl_value_t*)jl_abi_adapter_cache_type); + add_builtin("abi_adapters", (jl_value_t*)jl_abi_adapters); add_builtin("Symbol", (jl_value_t*)jl_symbol_type); add_builtin("SSAValue", (jl_value_t*)jl_ssavalue_type); add_builtin("SlotNumber", (jl_value_t*)jl_slotnumber_type); diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 713dc1be085a3..8a56dc76ce4cf 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -103,7 +103,7 @@ JL_DLLEXPORT void jl_jit_unregister_ci_fallback(jl_code_instance_t *ci) { } -JL_DLLEXPORT void *jl_create_native_fallback(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int cache, size_t world, jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_cis) UNAVAILABLE +JL_DLLEXPORT void *jl_create_native_fallback(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int cache, size_t world, jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_code) UNAVAILABLE JL_DLLEXPORT void *jl_emit_native_fallback(jl_array_t *codeinfos, jl_array_t *ci_order, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _external_linkage) UNAVAILABLE JL_DLLEXPORT void jl_dump_compiles_fallback(void *s) @@ -125,6 +125,17 @@ JL_DLLEXPORT jl_value_t *jl_dump_function_asm_fallback(jl_llvmf_dump_t* dump, ch JL_DLLEXPORT void jl_get_function_id_fallback(void *native_code, jl_code_instance_t *ncode, int32_t *func_idx, int32_t *specfunc_idx) UNAVAILABLE +JL_DLLEXPORT void jl_get_adapter_id_fallback(void *native_code, jl_abi_adapter_t *rec, + int32_t *adapter_idx) UNAVAILABLE + +// NULL = "this emission resolved no target": without codegen there is no emission, so the +// serializer writes every trampoline's `last_invokee` as unresolved. +JL_DLLEXPORT jl_value_t *jl_get_trampoline_invokee_fallback(void *native_code, jl_dispatch_trampoline_t *tr) +{ + (void)native_code; (void)tr; + return NULL; +} + JL_DLLEXPORT void *jl_get_llvm_function_fallback(void *native_code, uint32_t idx) UNAVAILABLE @@ -146,7 +157,22 @@ JL_DLLEXPORT uint64_t jl_getUnwindInfo_fallback(uint64_t dwAddr) JL_NOTSAFEPOINT JL_DLLEXPORT void jl_register_passbuilder_callbacks_fallback(void *PB) { } -JL_DLLEXPORT void *jl_jit_abi_converter_fallback(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst) UNAVAILABLE +JL_DLLEXPORT void *jl_jit_abi_converter_fallback(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst, jl_value_t **invokee) JL_CANSAFEPOINT +{ + // Without codegen we cannot JIT a fresh adapter, so use an existing one: the target's own + // specptr, an adapter compiled into a loaded image, or (the codeinst == NULL re-query) + // the signature's dynamic-dispatch adapter, which dispatches at call time and so stays + // correct when the resolved target shifts after load (julia#61949). + (void)ct; + void *f = jl_lookup_abi_converter(from_abi, codeinst, NULL, NULL, NULL, invokee); + if (f == NULL && codeinst != NULL) + f = jl_lookup_abi_converter(from_abi, NULL, NULL, NULL, NULL, invokee); + if (f != NULL) + return f; + jl_errorf("cfunction: no ABI adapter is available for this signature in a build without " + "codegen (none was compiled into a loaded image)"); + return NULL; +} //LLVM C api to the julia JIT JL_DLLEXPORT void* JLJITGetLLVMOrcExecutionSession_fallback(void* JIT) UNAVAILABLE diff --git a/src/datatype.c b/src/datatype.c index 3513502da5594..d5c0bd3052e5f 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -51,6 +51,19 @@ JL_DLLEXPORT jl_methcache_t *jl_new_method_cache(void) return mc; } +JL_DLLEXPORT jl_abi_adapter_cache_t *jl_new_abi_adapter_cache(void) +{ + jl_task_t *ct = jl_current_task; + jl_abi_adapter_cache_t *c = + (jl_abi_adapter_cache_t*)jl_gc_alloc(ct->ptls, sizeof(jl_abi_adapter_cache_t), + jl_abi_adapter_cache_type); + // The cache is a TypeMap; jl_nothing is the empty map. `writelock` is a trailing hidden + // field (not part of the datatype); JL_MUTEX_INIT fully initializes it (owner/count). + jl_atomic_store_relaxed(&c->cache, (jl_typemap_t*)jl_nothing); + JL_MUTEX_INIT(&c->writelock, "jl_abi_adapters->writelock"); + return c; +} + JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module) { jl_methcache_t *mc = jl_new_method_cache(); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 1ad2a788d4472..724d295519cfe 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -317,43 +317,12 @@ jl_emitted_output_t jl_codegen_output_t::finish(std::unique_ptr ctx // // If `codeinst` is NULL, the returned specptr instead performs a standard `apply_generic` // call via a dynamic dispatch. -extern "C" JL_DLLEXPORT_CODEGEN -void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi, - jl_code_instance_t *codeinst) -{ - void *target = nullptr; - bool target_specsig = false; - jl_callptr_t invoke = nullptr; - if (codeinst != nullptr) { - uint8_t specsigflags; - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - void *specptr = nullptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &specptr, /* waitcompile */ 1); - if (invoke != nullptr) { - if (invoke == jl_fptr_const_return_addr) { - target = nullptr; - target_specsig = false; - } - else if (invoke == jl_fptr_args_addr) { - assert(specptr != nullptr); - if (!from_abi.specsig && jl_subtype(codeinst->rettype, from_abi.rt)) - return specptr; // no adapter required - target = specptr; - target_specsig = false; - } - else if (specsigflags & JL_CI_FLAGS_SPECPTR_SPECIALIZED) { - assert(specptr != nullptr); - if (from_abi.specsig && jl_egal(mi->specTypes, from_abi.sigt) && jl_egal(codeinst->rettype, from_abi.rt)) - return specptr; // no adapter required - - target = specptr; - target_specsig = true; - } - } - } - - orc::ThreadSafeModule result_m; +// JIT the adapter thunk bridging `from_abi` to `codeinst` (via `target`/`invoke`), returning +// its address. +static void *jit_adapter(jl_abi_t from_abi, jl_code_instance_t *codeinst, + void *target, bool target_specsig, jl_callptr_t invoke) JL_CANSAFEPOINT +{ std::string gf_thunk_name; auto ctx = std::make_unique(); auto mod = jl_create_llvm_module("gfthunk", *ctx, jl_ExecutionEngine->getDataLayout(), @@ -390,6 +359,72 @@ void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi, return (void*)Addr; } +// Resolve + cache the ABI adapter for (from_abi, codeinst): one adapter is JIT'd per +// distinct key rather than one per call site. Some paths short-circuit to the CI's own +// specptr without a wrapping adapter (and so aren't cached). +static void *jl_get_abi_adapter(jl_abi_t from_abi, jl_code_instance_t *codeinst, + jl_value_t **invokee) JL_CANSAFEPOINT +{ + void *target = nullptr; + int target_specsig = 0; + jl_callptr_t invoke = nullptr; + void *f = jl_lookup_abi_converter(from_abi, codeinst, &target, &target_specsig, &invoke, invokee); + if (f != nullptr) + return f; + + // JIT outside the writelock: compilation is slow and must not serialize unrelated cache + // users. A concurrent resolution of the same key may compile a duplicate thunk; the + // recheck below then discards the loser's. + void *fptr = jit_adapter(from_abi, codeinst, target, target_specsig, invoke); + if (codeinst != nullptr && invoke == nullptr) { + // The target's entry points were unreadable (e.g. mid-publication, see the TODO in + // jl_read_codeinst_invoke), so this thunk was built from transient state: return it + // uncached rather than caching a permanently degraded record for the key. The + // caller holds it for as long as it needs; the next resolution of this key rebuilds + // the proper adapter. + return fptr; + } + JL_LOCK(&jl_abi_adapters->writelock); + jl_abi_adapter_t *e = jl_lookup_abi_adapter(from_abi.sigt, from_abi.rt, codeinst, + from_abi.specsig, from_abi.is_opaque_closure, from_abi.nargs); + JL_GC_PROMISE_ROOTED(e); // rooted by the cache + if (e != nullptr && e->fptr != nullptr) { + // lost the race: use the winner's thunk + JL_UNLOCK(&jl_abi_adapters->writelock); + if (invokee) + *invokee = (jl_value_t*)e; + return e->fptr; + } + if (e != nullptr) { + // A cached record without a compiled thunk (e.g. restored from user data with no + // fvar slot): fill it in place. Installing a fresh record instead would leave this + // one permanently shadowing the key. + e->fptr = fptr; + if (invokee) + *invokee = (jl_value_t*)e; + } + else { + jl_abi_adapter_t *entry = jl_new_abi_adapter(from_abi.sigt, from_abi.rt, codeinst, + from_abi.specsig, from_abi.is_opaque_closure, from_abi.nargs, fptr); + JL_GC_PUSH1(&entry); + // we hold the writelock (reentrant) and confirmed absence, so this inserts `entry` + entry = jl_insert_abi_adapter(entry); + if (invokee) + *invokee = (jl_value_t*)entry; + JL_GC_POP(); + } + JL_UNLOCK(&jl_abi_adapters->writelock); + return fptr; +} + +extern "C" JL_DLLEXPORT_CODEGEN +void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi, + jl_code_instance_t *codeinst, jl_value_t **invokee) JL_CANSAFEPOINT +{ + (void)ct; + return jl_get_abi_adapter(from_abi, codeinst, invokee); +} + // lock for places where only single threaded behavior is implemented, so we need GC support static jl_mutex_t jitlock; diff --git a/src/jitlayers.h b/src/jitlayers.h index d0b34155f881e..a7a3ddb30f3bb 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -462,6 +462,10 @@ class jl_codegen_output_t { SmallVector, 0> external_fns; SmallVector cfuncs; + // ABIAdapter records compiled for @cfunction/@ccallable serialization: each entry pairs a + // materialized ABIAdapter record with the Function emitted for its resolved target, so the + // serialized record's `fptr` is wired to that Function on load without a JIT. + SmallVector, 0> abi_adapter_records; std::map global_targets; jl_array_t *temporary_roots = nullptr; SmallSet temporary_roots_set; diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 47653939556ce..b3bd44a2fe174 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -2,6 +2,9 @@ // Pointers that are exposed through the public libjulia #define JL_EXPORTED_DATA_POINTERS(XX) \ + XX(abi_adapter_type, jl_datatype_t*) \ + XX(abi_adapter_cache_type, jl_datatype_t*) \ + XX(abi_adapters, jl_abi_adapter_cache_t*) \ XX(abioverride_type, jl_datatype_t*) \ XX(abstractarray_type, jl_unionall_t*) \ XX(abstractstring_type, jl_datatype_t*) \ diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 7b432dd64d745..bebf6eb8389e4 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -552,6 +552,7 @@ YY(jl_dump_fptr_asm) \ YY(jl_emit_native) \ YY(jl_get_function_id) \ + YY(jl_get_adapter_id) \ YY(jl_struct_to_llvm) \ YY(jl_type_to_llvm) \ YY(jl_getUnwindInfo) \ diff --git a/src/jltypes.c b/src/jltypes.c index 1691e09b9833b..c27b9c40adfc3 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3728,6 +3728,43 @@ void jl_init_types(void) JL_GC_DISABLED jl_value_t *pointer_void = jl_apply_type1((jl_value_t*)jl_pointer_type, (jl_value_t*)jl_nothing_type); jl_voidpointer_type = (jl_datatype_t*)pointer_void; + // One record of the ABI-adapter cache (jl_abi_adapters->cache). The C-side `specsig`/ + // `is_opaque_closure`/`nargs` bitfields pack into a single word exposed to Julia as + // `flags`. The field order must match `jl_abi_adapter_t` in julia.h exactly: the GC + // derives the pointer-field offsets from this datatype. + jl_abi_adapter_type = + jl_new_datatype(jl_symbol("ABIAdapter"), core, jl_any_type, jl_emptysvec, + jl_perm_symsvec(6, + "sigt", + "rt", + "flags", + "ci", + "fptr", + "next"), + jl_svec(6, + jl_any_type, // hash-consed Tuple type + jl_any_type, // hash-consed return type + jl_ulong_type, // packed specsig/is_opaque_closure/nargs + jl_any_type, // CodeInstance (NULL/#undef for dynamic-dispatch adapters) + pointer_void, // JITed adapter address + jl_any_type), // next record in the sigt bucket chain (may be NULL) + jl_emptysvec, + 0, 1, 3); + const static uint32_t abi_adapter_atomicfields[1] = { 0b100000 }; // next (field 5) + jl_abi_adapter_type->name->atomicfields = abi_adapter_atomicfields; + + // Singleton ABIAdapterCache type: holds the `adapters` TypeMap root (jl_nothing / + // TypeMapLevel / TypeMapEntry), replaced atomically on insert. The `jl_mutex_t writelock` + // is a trailing hidden C field (not registered here), following jl_method_type's writelock. + jl_abi_adapter_cache_type = + jl_new_datatype(jl_symbol("ABIAdapterCache"), core, jl_any_type, jl_emptysvec, + jl_perm_symsvec(1, "cache"), + jl_svec1(jl_any_type), // adapters: TypeMap root (jl_nothing when empty) + jl_emptysvec, + 0, 1, 1); + const static uint32_t abi_adapter_cache_atomicfields[1] = { 0b1 }; // adapters + jl_abi_adapter_cache_type->name->atomicfields = abi_adapter_cache_atomicfields; + tv = jl_svec2(tvar("T"), tvar("N")); jl_abstractarray_type = (jl_unionall_t*) jl_new_abstracttype((jl_value_t*)jl_symbol("AbstractArray"), core, @@ -3789,6 +3826,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_array_uint64_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint64_type, jl_box_long(1)); jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally jl_an_empty_memory_any = (jl_value_t*)jl_alloc_memory_any(0); // used internally + jl_abi_adapters = jl_new_abi_adapter_cache(); // process-global ABI-adapter cache // finish initializing module Core core = jl_core_module; diff --git a/src/julia.h b/src/julia.h index 529b2b25163ad..0ec087e680292 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1048,6 +1048,72 @@ typedef struct _jl_methtable_t { jl_genericmemory_t *backedges; // IdDict{top typenames, Vector{uncovered (sig => caller::CodeInstance)}} } jl_methtable_t; +// One record of the global ABI-adapter cache (`jl_abi_adapters->cache`): a JIT'd adapter +// thunk that bridges the caller ABI (from_abi) to a target CodeInstance. Cached so +// `jl_jit_abi_converter` emits one adapter per distinct key rather than one per call site. +typedef struct _jl_abi_adapter_t { + JL_DATA_TYPE + // caller ABI + jl_value_t *sigt; // hash-consed Julia type (Tuple type) + jl_value_t *rt; // hash-consed Julia type (return type) + size_t specsig : 1; // whether the caller passes args in specsig (vs. boxed) form + size_t is_opaque_closure : 1; // whether the adapter bridges an OpaqueClosure caller ABI + size_t nargs : 8 * sizeof(size_t) - 2; // number of arguments in `sigt` + + // callee target + jl_code_instance_t *ci; // target CodeInstance; may be NULL for dispatcher adapters + + // adapter function pointer + void *fptr; // JITed adapter address (not GC-tracked) + // Bucket chain: the adapters cache is a TypeMap keyed on `sigt`; records sharing one + // `sigt` entry are chained here and disambiguated by (rt, ci, specsig, is_opaque_closure, + // nargs). Append-only (the head in the TypeMap entry's value never changes), so the chain + // is safe to walk lock-free. NULL = tail. + _Atomic(struct _jl_abi_adapter_t*) next; +} jl_abi_adapter_t; + +// Latest-world dispatch trampoline record (`jl_dispatch_trampolines->cache`), used by +// @cfunction/@ccallable: holds a *signature*, not a fixed CodeInstance, and re-resolves the +// target as the world advances (the inline call site polls `last_world` vs the world counter +// and re-resolves via jl_resolve_trampoline). Mutable; `fptr`/`last_world` are atomic. +typedef struct _jl_dispatch_trampoline_t { + JL_DATA_TYPE + jl_value_t *sigt; // key: resolution sig Tuple{typeof(f), A...} (for jl_get_specialization1) + jl_value_t *rt; // key: declared return type (declrt for @cfunction) + // Last resolved target, as a Union{CodeInstance, ABIAdapter} (NULL = unresolved): a bare + // `CodeInstance` when the declared ABI matches its specptr (no adapter needed), otherwise + // the `ABIAdapter` record whose `fptr` is the compiled thunk. + jl_value_t *last_invokee; + _Atomic(void*) fptr; // redundant cached fptr (== last_invokee's specptr/adapter fptr) for the hot-path poll + _Atomic(size_t) last_world; // world for which `fptr`/`last_invokee` are valid (0 = unresolved) + uint8_t specsig; // key: 1 if `fptr` is a specsig adapter, 0 if jlcall (cgparams-dependent) + // Bucket chain: the trampolines cache is a TypeMap keyed on `sigt`; records sharing one + // `sigt` entry are chained here and disambiguated by (rt, specsig). Append-only, safe to + // walk lock-free. NULL = tail. + _Atomic(struct _jl_dispatch_trampoline_t*) next; +} jl_dispatch_trampoline_t; + +// Process-global cache of JIT'd ABI adapters. The singleton instance is `jl_abi_adapters` +// (bound in Core as `abi_adapters`). Holds a TypeMap keyed on `sigt`, guarded by its own +// `writelock`; reads are lock-free. `jl_nothing` is the empty cache. +typedef struct _jl_abi_adapter_cache_t { + JL_DATA_TYPE + _Atomic(jl_typemap_t*) cache; // (sigt) -> bucket of jl_abi_adapter_t +// hidden fields: + jl_mutex_t writelock; // guards mutation of the adapters cache +} jl_abi_adapter_cache_t; + +// Process-global cache of @cfunction/@ccallable dispatch trampolines. The singleton instance +// is `jl_dispatch_trampolines` (bound in Core as `dispatch_trampolines`). Holds a TypeMap keyed on +// `sigt`, guarded by its own `writelock` (separate from the adapter cache's); reads are +// lock-free. `jl_nothing` is the empty cache. +typedef struct _jl_dispatch_trampoline_cache_t { + JL_DATA_TYPE + _Atomic(jl_typemap_t*) cache; // (sigt) -> bucket of jl_dispatch_trampoline_t +// hidden fields: + jl_mutex_t writelock; // guards mutation of the trampolines cache +} jl_dispatch_trampoline_cache_t; + typedef struct { JL_DATA_TYPE jl_sym_t *head; @@ -1756,6 +1822,8 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_module(v) jl_typetagis(v,jl_module_tag<<4) #define jl_is_mtable(v) jl_typetagis(v,jl_methtable_type) #define jl_is_mcache(v) jl_typetagis(v,jl_methcache_type) +#define jl_is_dispatch_trampoline(v) jl_typetagis(v,jl_dispatch_trampoline_type) +#define jl_is_abi_adapter(v) jl_typetagis(v,jl_abi_adapter_type) #define jl_is_task(v) jl_typetagis(v,jl_task_tag<<4) #define jl_is_string(v) jl_typetagis(v,jl_string_tag<<4) #define jl_is_cpointer(v) jl_is_cpointer_type(jl_typeof(v)) diff --git a/src/julia_internal.h b/src/julia_internal.h index baf0f03c077b5..59e87440e2707 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1736,7 +1736,30 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals) JL_CANSAFEPOINT; JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data) JL_CANSAFEPOINT; -JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst) JL_CANSAFEPOINT; +// Returns the adapter fptr; `*invokee` (optional) receives the Union{CodeInstance, ABIAdapter} +// it was derived from. +JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst, jl_value_t **invokee) JL_CANSAFEPOINT; + +// ABI-adapter cache (src/abi_adapter.c). The bucketed TypeMap +// bookkeeping lives in libjulia-internal; the JIT half (jl_get_abi_adapter) is in codegen. +JL_DLLEXPORT jl_abi_adapter_cache_t *jl_new_abi_adapter_cache(void) JL_CANSAFEPOINT; +JL_DLLEXPORT int jl_abi_matches_invoke_api(jl_abi_t from_abi, jl_invoke_api_t api, + jl_method_instance_t *mi, jl_value_t *rettype) JL_CANSAFEPOINT; +// The target's own compiled entry point when it already satisfies `from_abi` (so no adapter +// is needed), else NULL. +JL_DLLEXPORT void *jl_abi_matching_specptr(jl_abi_t from_abi, jl_code_instance_t *codeinst) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_abi_adapter_t *jl_lookup_abi_adapter(jl_value_t *sigt, jl_value_t *rt, + jl_code_instance_t *ci, int specsig, int is_opaque_closure, size_t nargs) JL_CANSAFEPOINT; +// Find an existing adapter (or the target's own matching specptr) without JITing; NULL means +// none exists. `*invokee` (optional) receives the Union{CodeInstance, ABIAdapter} behind a +// non-NULL return. +JL_DLLEXPORT void *jl_lookup_abi_converter(jl_abi_t from_abi, jl_code_instance_t *codeinst, + void **target, int *target_specsig, jl_callptr_t *invoke, jl_value_t **invokee) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_abi_adapter_t *jl_new_abi_adapter(jl_value_t *sigt, jl_value_t *rt, + jl_code_instance_t *ci, int specsig, int is_opaque_closure, size_t nargs, void *fptr) JL_CANSAFEPOINT; +// Insert the record if its key is absent and return the canonical record; keep-first, like +// jl_specializations_get_or_insert. +JL_DLLEXPORT jl_abi_adapter_t *jl_insert_abi_adapter(jl_abi_adapter_t *e) JL_CANSAFEPOINT; // Special filenames used to refer to internal julia libraries @@ -2226,7 +2249,7 @@ JL_DLLIMPORT jl_value_t *jl_dump_function_ir(jl_llvmf_dump_t *dump, char strip_i JL_DLLIMPORT jl_value_t *jl_dump_function_asm(jl_llvmf_dump_t *dump, char emit_mc, const char* asm_variant, const char *debuginfo, char binary, char raw) JL_CANSAFEPOINT; typedef jl_value_t *(*jl_codeinstance_lookup_t)(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); -JL_DLLIMPORT void *jl_create_native(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int cache, size_t world, jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_cis) JL_CANSAFEPOINT; +JL_DLLIMPORT void *jl_create_native(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int cache, size_t world, jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_code) JL_CANSAFEPOINT; JL_DLLIMPORT void *jl_emit_native(jl_array_t *codeinfos, jl_array_t *ci_order, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _external_linkage) JL_CANSAFEPOINT; JL_DLLIMPORT void jl_dump_native(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, @@ -2237,6 +2260,8 @@ JL_DLLIMPORT void jl_get_llvm_external_fns(void *native_code, size_t *num_els, jl_code_instance_t *fns); JL_DLLIMPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, int32_t *func_idx, int32_t *specfunc_idx); +JL_DLLIMPORT void jl_get_adapter_id(void *native_code, jl_abi_adapter_t *rec, + int32_t *adapter_idx); JL_DLLIMPORT void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_code_instance_t **linfos, size_t n); JL_DLLIMPORT void jl_get_llvm_cis(void *native_code, size_t *num_els, diff --git a/src/runtime_ccall.c b/src/runtime_ccall.c index 0972f21bec036..ff55e242d745f 100644 --- a/src/runtime_ccall.c +++ b/src/runtime_ccall.c @@ -430,7 +430,7 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) if (codeinst == NULL) { // Generate an adapter to a dynamic dispatch if (cfuncdata->unspecialized == NULL) - cfuncdata->unspecialized = jl_jit_abi_converter(ct, from_abi, NULL); + cfuncdata->unspecialized = jl_jit_abi_converter(ct, from_abi, NULL, NULL); f = cfuncdata->unspecialized; } else { @@ -442,7 +442,7 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data) // even though we're likely to encounter memory errors in that case jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance((jl_method_instance_t*)mi)); } - f = jl_jit_abi_converter(ct, from_abi, codeinst); + f = jl_jit_abi_converter(ct, from_abi, codeinst, NULL); } cfuncdata->plast_codeinst = &cfuncdata->last_codeinst; diff --git a/src/staticdata.c b/src/staticdata.c index e2a72cc0dfdef..c10833dbe75a6 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -737,6 +737,19 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } } } + if (jl_typetagis(v, jl_abi_adapter_cache_type)) { + // The adapter cache is process-local JIT state (its `fptr`s are process addresses); + // serialize it empty. Any adapter that must survive stays reachable through the code + // that references it and is re-inserted on load, not via this cache. + jl_abi_adapter_cache_t *c = (jl_abi_adapter_cache_t*)v; + record_field_change((jl_value_t**)&c->cache, jl_nothing); + } + if (jl_is_abi_adapter(v)) { + // Like the cache itself, the adapter's `sigt` bucket chain is process-local and + // rebuilt on load (jl_insert_abi_adapter), so drop `next`. + jl_abi_adapter_t *ad = (jl_abi_adapter_t*)v; + record_field_change((jl_value_t**)&ad->next, NULL); + } if (jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; jl_method_instance_t *mi = jl_get_ci_mi(ci); @@ -1816,6 +1829,38 @@ static void jl_write_values(jl_serializer_state *s) JL_CANSAFEPOINT JL_GC_DISABL arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + BuiltinFunctionTag + builtin_id - 2)); // relocation target } } + else if (jl_is_abi_adapter(v)) { + assert(f == s->s); + // The compiled adapter thunk lives in the image as an fvar. Null the record's + // process-local `fptr` and record the wiring in `fptr_record` keyed on that fvar + // id, so jl_update_all_fptrs sets `fptr` from the fvar on load (parallels the + // CodeInstance specptr; CI vs record disambiguated by the object's type tag). + jl_abi_adapter_t *newad = (jl_abi_adapter_t*)&f->buf[reloc_offset]; + newad->fptr = NULL; + int32_t adapter_id = 0; + jl_get_adapter_id(native_functions, (jl_abi_adapter_t*)v, &adapter_id); + if (adapter_id > 0) { + ios_ensureroom(s->fptr_record, adapter_id * sizeof(void*)); + ios_seek(s->fptr_record, (adapter_id - 1) * sizeof(void*)); + write_reloc_t(s->fptr_record, reloc_offset); +#ifdef _P64 + if (sizeof(reloc_t) < 8) + write_padding(s->fptr_record, 8 - sizeof(reloc_t)); +#endif + // Re-insert this record into the running adapters cache on load. Records + // with no fvar slot (adapter_id == 0, e.g. a runtime-created record + // reached through user data) have no thunk to wire, so they serialize as + // inert data and are never published to the cache. + arraylist_push(&s->fixup_objs, (void*)reloc_offset); + } + } + else if (jl_typetagis(v, jl_abi_adapter_cache_type)) { + assert(f == s->s); + // The cache's `writelock` is a hidden trailing C field (not a datatype field); + // pad the serialized object out to the full struct size so the restored lock + // is zero-initialized (mirrors the jl_method_t writelock handling above). + write_padding(f, sizeof(jl_abi_adapter_cache_t) - tot); + } else if (jl_is_datatype(v)) { assert(f == s->s); jl_datatype_t *dt = (jl_datatype_t*)v; @@ -2264,10 +2309,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) specfunc = 0; offset = ~offset; } - jl_code_instance_t *codeinst = (jl_code_instance_t*)(base + offset); - assert(jl_is_method(jl_get_ci_mi(codeinst)->def.method) && jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return); - assert(specfunc ? jl_atomic_load_relaxed(&codeinst->invoke) != NULL : jl_atomic_load_relaxed(&codeinst->invoke) == NULL); - linfos[i] = codeinst; + void *obj = (void*)(base + offset); void *fptr = fvars.ptrs[i]; for (; clone_idx < fvars.nclones; clone_idx++) { uint32_t idx = fvars.clone_idxs[clone_idx] & jl_sysimg_val_mask; @@ -2277,6 +2319,22 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) fptr = fvars.clone_ptrs[clone_idx]; break; } + if (jl_typeof((jl_value_t*)obj) == (jl_value_t*)jl_abi_adapter_type) { + // @cfunction/@ccallable ABI-adapter record: wire the compiled adapter thunk into + // the record's `fptr` (disambiguated from a CodeInstance by the object's type + // tag). A consumer that points at this record reads this `fptr` on its first + // post-load call, re-resolving against the current world only if a redefinition + // shifted dispatch since image build. + jl_abi_adapter_t *ad = (jl_abi_adapter_t*)obj; + assert(specfunc); // adapter slots are written without the ~offset inversion + assert(ad->fptr == NULL && fptr != NULL); + ad->fptr = fptr; + continue; + } + jl_code_instance_t *codeinst = (jl_code_instance_t*)obj; + assert(jl_is_method(jl_get_ci_mi(codeinst)->def.method) && jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return); + assert(specfunc ? jl_atomic_load_relaxed(&codeinst->invoke) != NULL : jl_atomic_load_relaxed(&codeinst->invoke) == NULL); + linfos[i] = codeinst; if (specfunc) { uint8_t flags = jl_atomic_load_relaxed(&codeinst->flags); flags |= JL_CI_FLAGS_INVOKE_MATCHES_SPECPTR | JL_CI_FLAGS_FROM_IMAGE; @@ -3065,8 +3123,17 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, if (extext_methods) { // Queue method extensions jl_queue_for_serialization(&s, extext_methods); - // Queue the new specializations - jl_queue_for_serialization(&s, new_ext); + } + // Queue `new_ext` (external CodeInstances + reported ABI adapters). Incremental images + // read the array back as a root (the external-method-cache CIs); the full/AOT image does + // not, so there we queue only the elements for reachability -- each is written and handled + // per-type by the load fixup (adapters -> jl_insert_abi_adapter), no array root needed. + if (new_ext) { + if (s.incremental) + jl_queue_for_serialization(&s, new_ext); + else + for (size_t i = 0; i < jl_array_nrows(new_ext); i++) + jl_queue_for_serialization(&s, jl_array_ptr_ref(new_ext, i)); } jl_serialize_reachable(&s); // step 1.2: ensure all gvars are part of the sysimage too @@ -3359,20 +3426,20 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli ff = f; } - jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_ext = NULL, *ext_foreign_cis = NULL; + jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_ext = NULL, *ext_foreign_code = NULL; int64_t checksumpos = 0; int64_t checksumpos_ff = 0; int64_t datastartpos = 0; - JL_GC_PUSH4(&mod_array, &extext_methods, &new_ext, &ext_foreign_cis); + JL_GC_PUSH4(&mod_array, &extext_methods, &new_ext, &ext_foreign_code); - ext_foreign_cis = jl_alloc_vec_any(0); + ext_foreign_code = jl_alloc_vec_any(0); mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) if (worklist) { if (_native_data != NULL) { if (suppress_precompile) newly_inferred = NULL; - *_native_data = jl_create_native(NULL, 0, 1, jl_atomic_load_acquire(&jl_world_counter), NULL, suppress_precompile ? (jl_array_t*)jl_an_empty_vec_any : worklist, 0, module_init_order, ext_foreign_cis); + *_native_data = jl_create_native(NULL, 0, 1, jl_atomic_load_acquire(&jl_world_counter), NULL, suppress_precompile ? (jl_array_t*)jl_an_empty_vec_any : worklist, 0, module_init_order, ext_foreign_code); } jl_write_header_for_incremental(f, worklist, mod_array, udeps, srctextpos, &checksumpos); if (emit_split) { @@ -3386,7 +3453,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } } else if (_native_data != NULL) { - *_native_data = jl_create_native(NULL, jl_options.trim, 0, jl_atomic_load_acquire(&jl_world_counter), mod_array, NULL, jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL, module_init_order, ext_foreign_cis); + *_native_data = jl_create_native(NULL, jl_options.trim, 0, jl_atomic_load_acquire(&jl_world_counter), mod_array, NULL, jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL, module_init_order, ext_foreign_code); } if (_native_data != NULL) native_functions = *_native_data; @@ -3399,7 +3466,9 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli assert((ct->reentrant_timing & 0b1110) == 0); ct->reentrant_timing |= 0b1000; if (worklist) { - // new_exts: [code_instances, ...], anything to add to the image just for side-effects + // new_ext: [code_instance | abi_adapter, ...], code to add to the image for side-effects + // (external CodeInstances, plus ABI adapters reported via ext_foreign_code that + // nothing else owns). Elements are handled per-type by the load fixup, not iterated as CIs. // Save the inferred code from newly inferred, external methods arraylist_t CIs; arraylist_new(&CIs, 0); @@ -3419,11 +3488,11 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli jl_array_ptr_1d_push(new_ext, (jl_value_t*)ci); } arraylist_free(&CIs); - // Merge foreign & external CIs - size_t n_ext = jl_array_nrows(ext_foreign_cis); + // Merge foreign & external code (external CodeInstances + reported ABI adapters) + size_t n_ext = jl_array_nrows(ext_foreign_code); for (size_t i = 0; i < n_ext; i++) - jl_array_ptr_1d_push(new_ext, jl_array_ptr_ref(ext_foreign_cis, i)); - ext_foreign_cis = NULL; // not needed anymore, free it + jl_array_ptr_1d_push(new_ext, jl_array_ptr_ref(ext_foreign_code, i)); + ext_foreign_code = NULL; // not needed anymore, free it // Collect method extensions // extext_methods: [method1, ...], worklist-owned "extending external" methods added to functions owned by modules outside the worklist @@ -3439,6 +3508,18 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } datastartpos = ios_pos(ff); } + else if (ext_foreign_code != NULL && jl_array_nrows(ext_foreign_code) > 0) { + // Non-worklist (full / AOT / --trim) image: the incremental `new_ext` machinery above is + // skipped, so report the compiled ABI adapters here as extra image code. They have no + // trampoline/code owner of their own, so this is what roots them through the pre-dump GC + // and gets them serialized + reinterned on load (see jl_create_native_impl's + // ext_foreign_code reporting; each is fvar-wired and re-inserted on load). + new_ext = jl_alloc_vec_any(0); + size_t n_ext = jl_array_nrows(ext_foreign_code); + for (size_t i = 0; i < n_ext; i++) + jl_array_ptr_1d_push(new_ext, jl_array_ptr_ref(ext_foreign_code, i)); + } + ext_foreign_code = NULL; // consumed jl_query_cache query_cache; init_query_cache(&query_cache); @@ -4176,6 +4257,10 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, o->bits.in_image = 1; } arraylist_free(&cleanup_list); + // Restored ABI-adapter records, re-inserted into the running cache after + // jl_update_all_fptrs below (see the comment there). + arraylist_t reinsert_objs; + arraylist_new(&reinsert_objs, 0); for (size_t i = 0; i < s.fixup_objs.len; i++) { uintptr_t item = (uintptr_t)s.fixup_objs.items[i]; jl_value_t *obj = (jl_value_t*)(image_base + item); @@ -4183,6 +4268,10 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_ptr_1d_push(*internal_methods, obj); assert(s.incremental); } + else if (jl_is_abi_adapter(obj)) { + // Deferred (see reinsert_objs above): re-inserted after jl_update_all_fptrs. + arraylist_push(&reinsert_objs, (void*)obj); + } else if (jl_is_method_instance(obj)) { jl_method_instance_t *newobj = jl_specializations_get_or_insert((jl_method_instance_t*)obj); assert(newobj == (jl_method_instance_t*)obj); // strict insertion expected @@ -4288,6 +4377,15 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_update_all_fptrs(&s, image); // fptr relocs and registration s.s = NULL; + // Re-insert the restored ABI-adapter records into the running cache, so matching + // consumers share them (and reuse the compiled adapters without a JIT). This publishes + // the records to lock-free readers, so it must happen only now, after + // jl_update_all_fptrs wired each record's `fptr`: a reader must never observe a NULL + // `fptr`. + for (size_t i = 0; i < reinsert_objs.len; i++) + jl_insert_abi_adapter((jl_abi_adapter_t*)reinsert_objs.items[i]); + arraylist_free(&reinsert_objs); + ios_close(&fptr_record); ios_close(&sysimg); diff --git a/src/typemap.c b/src/typemap.c index a89c46cb8df74..1ea68b00d36b2 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -424,6 +424,24 @@ int jl_typemap_visitor(jl_typemap_t *cache, jl_typemap_visitor_fptr fptr, void * } } +static int typemap_count_visitor(jl_typemap_entry_t *l, void *closure) JL_NOTSAFEPOINT +{ + (void)l; + (*(size_t*)closure)++; + return 1; +} + +// Number of entries in a TypeMap (`jl_nothing` => 0). Note: for caches that chain +// multiple records per entry (e.g. the ABI-adapter bucket chains), this counts +// distinct entries, not total records. +JL_DLLEXPORT size_t jl_typemap_count(jl_typemap_t *cache) JL_CANSAFEPOINT +{ + size_t n = 0; + if (cache != NULL && (jl_value_t*)cache != jl_nothing) + jl_typemap_visitor(cache, typemap_count_visitor, (void*)&n); + return n; +} + // ---- jl_typemap_list_t: sigt-keyed record lists ---- // Shared machinery for caches that key a TypeMap *exactly* on a signature type and hold // several records per entry (the ABI-adapter and dispatch-trampoline caches): the entry's From e157bedb37188787116ff0c6e60ef8a0fdfc7012 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 1 Jul 2026 13:37:21 -0400 Subject: [PATCH 3/5] codegen: add DispatchTrampoline cache and resolver Add the DispatchTrampoline datatype and a process-global trampoline cache (`Core.dispatch_trampolines`, a DispatchTrampolineCache with its own writelock), interning one @cfunction/@ccallable dispatch record per (sigt, rt). A trampoline holds `last_invoked` -- a Union{CodeInstance, ABIAdapter} recording its resolved target -- plus a redundant `fptr` cache for the hot-path poll. Trampolines serialize alongside their adapters: the record's bucket `next` is reset and it is re-interned on load (jl_reintern_trampoline); `last_invoked` serializes as a normal reference so the first post-load call restores `fptr` straight from it. The resolver itself (jl_resolve_trampoline) lands with the @cfunction migration. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Claude Fable 5 --- base/methodshow.jl | 2 + src/Makefile | 2 +- src/aotcompile.cpp | 214 ++++++++++++++++++++++++-------------- src/builtins.c | 6 ++ src/datatype.c | 11 ++ src/dispatch_trampoline.c | 102 ++++++++++++++++++ src/jl_exported_data.inc | 3 + src/jltypes.c | 39 +++++++ src/julia.h | 2 +- src/julia_internal.h | 9 +- src/staticdata.c | 52 +++++++-- 11 files changed, 352 insertions(+), 90 deletions(-) create mode 100644 src/dispatch_trampoline.c diff --git a/base/methodshow.jl b/base/methodshow.jl index 8dd53ef498046..74748d292b2f7 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -389,6 +389,8 @@ show(io::IO, ::MIME"text/plain", ms::MethodList) = show_method_table(io, ms) show(io::IO, mt::Core.MethodTable) = print(io, mt.module, ".", mt.name, " is a Core.MethodTable with ", length(mt), " methods.") show(io::IO, c::Core.ABIAdapterCache) = print(io, "Core.ABIAdapterCache with ", ccall(:jl_typemap_count, Csize_t, (Any,), getfield(c, :cache)), " entries.") +show(io::IO, c::Core.DispatchTrampolineCache) = print(io, "Core.DispatchTrampolineCache with ", + ccall(:jl_typemap_count, Csize_t, (Any,), getfield(c, :cache)), " entries.") function inbase(m::Module) if m == Base diff --git a/src/Makefile b/src/Makefile index 89e5172580633..e480eb0a1ebcc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -67,7 +67,7 @@ SRCS := \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ threading scheduler stackwalk null_sysimage \ method jlapi signal-handling safepoint timing subtype rtutils \ - crc32c APInt processor ircode opaque_closure abi_adapter codegen-stubs coverage runtime_ccall engine \ + crc32c APInt processor ircode opaque_closure abi_adapter dispatch_trampoline codegen-stubs coverage runtime_ccall engine \ $(GC_SRCS) RT_LLVMLINK := diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 9c1da209d0068..3b6b21190d72b 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -184,6 +184,9 @@ typedef struct { // records share an adapter Function -- exactly as CodeInstances get a fvar slot each. std::map pending_adapters; // emitted, maybe-unregistered std::map adapter_fvar_map; // record -> fvar index + // trampoline -> AOT-resolved Union{CodeInstance, ABIAdapter}; consulted by the serializer + // (jl_get_trampoline_invokee) when writing a trampoline's last_invokee. + std::map tramp_invokee_map; } jl_native_code_desc_t; extern "C" JL_DLLEXPORT_CODEGEN @@ -234,6 +237,19 @@ void jl_get_adapter_id_impl(void *native_code, jl_abi_adapter_t *rec, int32_t *a *adapter_idx = idx; } +// The AOT-resolved target for dispatch trampoline `tr`: the Union{CodeInstance, ABIAdapter} +// this emission compiled for it, or NULL if it resolved no target for `tr`. Consulted by the +// serializer when writing a trampoline's `last_invokee`. +extern "C" JL_DLLEXPORT_CODEGEN +jl_value_t *jl_get_trampoline_invokee_impl(void *native_code, jl_dispatch_trampoline_t *tr) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + if (!data) + return NULL; + auto it = data->tramp_invokee_map.find(tr); + return it == data->tramp_invokee_map.end() ? NULL : it->second; +} + extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_cis_impl(void *native_code, size_t *num_elements, jl_code_instance_t **data) { @@ -582,8 +598,91 @@ static Function *aot_abi_converter(jl_codegen_output_t &out, jl_abi_t from_abi, return F; } +// AOT counterpart of jl_resolve_trampoline: resolve the target for `tr` at the build world, +// emit its ABI adapter, and record the materialized ABIAdapter record in +// `out.tramp_invokees` / `out.adapter_funcs`. When the target's compiled ABI already +// satisfies the declared C ABI, `tramp_invokees` maps to the bare `CodeInstance` instead and +// no adapter is emitted. +static void emit_trampoline_adapter(jl_codegen_output_t &out, jl_dispatch_trampoline_t *tr, + jl_abi_t from_abi, DenseMap &compiled_mi, + size_t latestworld) JL_CANSAFEPOINT +{ + JL_GC_PROMISE_ROOTED(tr); + jl_value_t *sigt = from_abi.sigt; + JL_GC_PROMISE_ROOTED(sigt); + jl_value_t *declrt = from_abi.rt; + JL_GC_PROMISE_ROOTED(declrt); + jl_method_instance_t *mi = (jl_method_instance_t*)jl_get_specialization1((jl_tupletype_t*)sigt, latestworld); + jl_code_instance_t *codeinst = nullptr; + if ((jl_value_t*)mi != jl_nothing) { + auto it = compiled_mi.find(mi); + if (it != compiled_mi.end()) + codeinst = it->second; + } + Function *F; + if (codeinst) { + JL_GC_PROMISE_ROOTED(codeinst); + jl_value_t *astrt = codeinst->rettype; + if (astrt != (jl_value_t*)jl_bottom_type && + jl_type_intersection(astrt, declrt) == jl_bottom_type) { + // Do not warn if the function never returns since it is occasionally required by + // the C API (typically error callbacks) even though we're likely to encounter + // memory errors in that case. + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); + } + const auto &decls = out.ci_funcs.find(codeinst)->second; + // If the target's own compiled ABI already satisfies the declared C ABI, no adapter + // is needed: map to the bare CodeInstance so the first post-load call derives `fptr` + // from its (independently wired) specptr. + if (jl_abi_matches_invoke_api(from_abi, decls.invoke_api, mi, codeinst->rettype)) { + out.tramp_invokees[tr] = (jl_value_t*)codeinst; // bare CI: no adapter thunk emitted + return; + } + if (decls.invoke_api == JL_INVOKE_CONST) { + std::string n = emit_abi_constreturn(out, from_abi, codeinst->rettype_const); + F = out.get_module().getFunction(n); + assert(F); + } + else if (decls.invoke_api == JL_INVOKE_SPARAM) { + // no specptr prototype declared; route through jl_invoke via the dispatcher + F = aot_abi_converter(out, from_abi, codeinst, nullptr, nullptr, false); + } + else if (decls.invoke_api == JL_INVOKE_ARGS) { + assert(decls.specptr); + F = aot_abi_converter(out, from_abi, codeinst, nullptr, decls.specptr, false); + } + else { + assert(decls.specptr); + F = aot_abi_converter(out, from_abi, codeinst, nullptr, decls.specptr, true); + } + } + else { + // no method at the build world: emit the shared dynamic-dispatch adapter + F = aot_abi_converter(out, from_abi, nullptr, nullptr, nullptr, false); + } + // A real thunk was needed: materialize the ABIAdapter record (its `fptr` is wired from the + // fvar on load) and map the trampoline to it. The record reaches the image via the + // serializer's `last_invokee` override (and ext_foreign_code); the first post-load call + // restores `fptr` straight from it. + jl_abi_adapter_t *rec = jl_new_abi_adapter_record(sigt, declrt, codeinst, + from_abi.specsig, from_abi.is_opaque_closure, from_abi.nargs, /*fptr*/nullptr); + // Nothing else roots the record until it is reported to ext_foreign_code. + JL_GC_PUSH1(&rec); + jl_array_ptr_1d_push(out.temporary_roots, (jl_value_t*)rec); + JL_GC_POP(); + out.tramp_invokees[tr] = (jl_value_t*)rec; + out.adapter_funcs.push_back({rec, F}); +} + +// Emit the adapter for each @cfunction/@ccallable dispatch trampoline (kind=STD, so the +// adapter sig is the call sig as-is). Replaces the old cfuncdata-array fill; the serializer +// wires the adapter into the image trampoline's `last_invokee` (jl_get_trampoline_invokee). +// Call sites sharing one interned trampoline (same sigt/rt/specsig) are deduplicated here, so +// each trampoline yields exactly one adapter record and one unspecialized record. static void generate_cfunc_thunks(jl_codegen_output_t &out) JL_CANSAFEPOINT { + if (out.cfuncs.empty()) + return; DenseMap compiled_mi; for (auto &[ci, _] : out.ci_funcs) { jl_method_instance_t *mi = jl_get_ci_mi(ci); @@ -593,77 +692,25 @@ static void generate_cfunc_thunks(jl_codegen_output_t &out) JL_CANSAFEPOINT } size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : out.cfuncs) { - jl_value_t *sigt = cfunc.abi.sigt; - JL_GC_PROMISE_ROOTED(sigt); - jl_value_t *declrt = cfunc.abi.rt; - JL_GC_PROMISE_ROOTED(declrt); - Function *unspec = aot_abi_converter(out, cfunc.abi, nullptr, nullptr, nullptr, false); - jl_code_instance_t *codeinst = nullptr; - auto assign_fptr = [&out, &cfunc, &codeinst, &unspec](Function *f) JL_CANSAFEPOINT { - ConstantArray *init = cast(cfunc.cfuncdata->getInitializer()); - SmallVector initvals; - for (unsigned i = 0; i < init->getNumOperands(); ++i) - initvals.push_back(init->getOperand(i)); - assert(initvals.size() == 8); - assert(initvals[0]->isNullValue()); - assert(initvals[2]->isNullValue()); - if (codeinst) { - Constant *llvmcodeinst = literal_pointer_val_slot(out, (jl_value_t*)codeinst); - initvals[2] = llvmcodeinst; // plast_codeinst - } - assert(initvals[4]->isNullValue()); - initvals[4] = unspec; - initvals[0] = f; - cfunc.cfuncdata->setInitializer(ConstantArray::get(init->getType(), initvals)); - }; - jl_method_instance_t *mi = (jl_method_instance_t*)jl_get_specialization1((jl_tupletype_t*)sigt, latestworld); - Function *func = nullptr; - if ((jl_value_t*)mi != jl_nothing) { - auto it = compiled_mi.find(mi); - if (it != compiled_mi.end()) { - codeinst = it->second; - JL_GC_PROMISE_ROOTED(codeinst); - const auto &decls = out.ci_funcs.find(codeinst)->second; - jl_value_t *astrt = codeinst->rettype; - if (astrt != (jl_value_t*)jl_bottom_type && - jl_type_intersection(astrt, declrt) == jl_bottom_type) { - // Do not warn if the function never returns since it is - // occasionally required by the C API (typically error callbacks) - // even though we're likely to encounter memory errors in that case - jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); - } - if (decls.invoke_api == JL_INVOKE_CONST) { - std::string gf_thunk_name = emit_abi_constreturn(out, cfunc.abi, codeinst->rettype_const); - auto F = out.get_module().getFunction(gf_thunk_name); - assert(F); - assign_fptr(F); - continue; - } - else if (decls.invoke_api == JL_INVOKE_ARGS) { - assert(decls.specptr); - if (!cfunc.abi.specsig && jl_subtype(astrt, declrt)) { - assign_fptr(decls.specptr); - continue; - } - assign_fptr(aot_abi_converter(out, cfunc.abi, codeinst, nullptr, decls.specptr, false)); - continue; - } - else if (decls.invoke_api == JL_INVOKE_SPARAM) { - func = nullptr; // use jl_invoke instead for these, since we don't declare these prototypes - } - else { - assert(decls.specptr); - if (jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt)) { - assign_fptr(decls.specptr); - continue; - } - assign_fptr(aot_abi_converter(out, cfunc.abi, codeinst, func, decls.specptr, true)); - continue; - } - } + if (out.tramp_invokees.count(cfunc.tramp)) + continue; // another call site already emitted this trampoline's adapter + emit_trampoline_adapter(out, cfunc.tramp, cfunc.abi, compiled_mi, latestworld); + // Outside --trim, also emit the dynamic-dispatch ("unspecialized", ci == NULL) + // adapter for this signature, so a codegen-free run can dispatch this + // @cfunction/@ccallable even when the resolved target shifts (julia#61949). No + // trampoline owns it; it survives via the ext_foreign_code report (see + // jl_create_native_impl). Under --trim, dynamic dispatch is unavailable, so skip it. + if (!jl_options.trim) { + Function *uf = aot_abi_converter(out, cfunc.abi, nullptr, nullptr, nullptr, false); + jl_abi_adapter_t *urec = jl_new_abi_adapter_record(cfunc.abi.sigt, cfunc.abi.rt, + /*ci*/nullptr, cfunc.abi.specsig, cfunc.abi.is_opaque_closure, cfunc.abi.nargs, + /*fptr*/nullptr); + JL_GC_PUSH1(&urec); + // Nothing else roots the record until it is reported to ext_foreign_code. + jl_array_ptr_1d_push(out.temporary_roots, (jl_value_t*)urec); + JL_GC_POP(); + out.adapter_funcs.push_back({urec, uf}); } - Function *f = codeinst ? aot_abi_converter(out, cfunc.abi, codeinst, func, nullptr, false) : unspec; - assign_fptr(f); } } @@ -732,13 +779,23 @@ void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int ex auto data = (jl_native_code_desc_t *)jl_emit_native((jl_array_t*)codeinfos, (jl_array_t*)ci_order, llvmmod, NULL, external_linkage ? 1 : 0); // Report each compiled ABI adapter as extra image code (via `ext_foreign_code`, like an // external CodeInstance), which roots it through the pre-dump GC and gets it serialized + - // reinterned on load even when nothing else owns it. The records are kept alive across - // emission in `data->out->temporary_roots`; re-root that array so the pushes are GC-safe. - if (ext_foreign_code && data->out && data->out->temporary_roots) { - jl_array_t *troots = data->out->temporary_roots; - JL_GC_PUSH1(&troots); + // reinterned on load even when nothing else owns it (in particular the `unspecialized` + // ci == NULL records). + // + // Invariant: the records are only reachable through `data->pending_adapters` (raw + // pointers) here -- they were de-rooted when jl_emit_native cleared + // `out.temporary_roots`, and stay live only because no safepoint runs between there and + // this point. Snapshot them into rooted locals before the pushes, since + // jl_array_ptr_1d_push may grow (and GC) the array. + if (ext_foreign_code && !data->pending_adapters.empty()) { + size_t nrec = data->pending_adapters.size(); + jl_value_t **recs; + JL_GC_PUSHARGS(recs, nrec); + size_t ri = 0; for (auto &kv : data->pending_adapters) - jl_array_ptr_1d_push(ext_foreign_code, (jl_value_t*)kv.first); + recs[ri++] = (jl_value_t*)kv.first; + for (size_t i = 0; i < nrec; i++) + jl_array_ptr_1d_push(ext_foreign_code, recs[i]); JL_GC_POP(); } JL_GC_POP(); @@ -1057,8 +1114,11 @@ static void jl_emit_native_to_output(jl_native_code_desc_t *data, jl_array_t *co // jl_get_adapter_id when the serializer reaches each record). Reporting them to // `ext_foreign_code` (so they're rooted + serialized even when nothing else owns them) is // done by the caller (jl_create_native_impl), which has that array in scope. - for (auto &[rec, F] : out.abi_adapter_records) + for (auto &[rec, F] : out.adapter_funcs) data->pending_adapters[rec] = F; + // Publish the trampoline -> AOT-resolved-target map for the serializer. + for (auto &[tr, invokee] : out.tramp_invokees) + data->tramp_invokee_map[tr] = invokee; } // also be used by extern consumers like GPUCompiler.jl to obtain a module containing diff --git a/src/builtins.c b/src/builtins.c index b2cec38bea5a5..8e4f129edfe02 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2720,7 +2720,13 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("TypeMapLevel", (jl_value_t*)jl_typemap_level_type); add_builtin("ABIAdapter", (jl_value_t*)jl_abi_adapter_type); add_builtin("ABIAdapterCache", (jl_value_t*)jl_abi_adapter_cache_type); + add_builtin("DispatchTrampolineCache", (jl_value_t*)jl_dispatch_trampoline_cache_type); + // DispatchTrampoline must be bound here explicitly: unlike the ABIAdapter* types it has no + // instances reachable at sysimage-save time (the cache tables are emptied on save), so + // nothing else would keep the type reachable for serialization. + add_builtin("DispatchTrampoline", (jl_value_t*)jl_dispatch_trampoline_type); add_builtin("abi_adapters", (jl_value_t*)jl_abi_adapters); + add_builtin("dispatch_trampolines", (jl_value_t*)jl_dispatch_trampolines); add_builtin("Symbol", (jl_value_t*)jl_symbol_type); add_builtin("SSAValue", (jl_value_t*)jl_ssavalue_type); add_builtin("SlotNumber", (jl_value_t*)jl_slotnumber_type); diff --git a/src/datatype.c b/src/datatype.c index d5c0bd3052e5f..920367d9d3463 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -64,6 +64,17 @@ JL_DLLEXPORT jl_abi_adapter_cache_t *jl_new_abi_adapter_cache(void) return c; } +JL_DLLEXPORT jl_dispatch_trampoline_cache_t *jl_new_dispatch_trampoline_cache(void) +{ + jl_task_t *ct = jl_current_task; + jl_dispatch_trampoline_cache_t *c = + (jl_dispatch_trampoline_cache_t*)jl_gc_alloc(ct->ptls, sizeof(jl_dispatch_trampoline_cache_t), + jl_dispatch_trampoline_cache_type); + jl_atomic_store_relaxed(&c->cache, (jl_typemap_t*)jl_nothing); + JL_MUTEX_INIT(&c->writelock, "jl_dispatch_trampolines->writelock"); + return c; +} + JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module) { jl_methcache_t *mc = jl_new_method_cache(); diff --git a/src/dispatch_trampoline.c b/src/dispatch_trampoline.c new file mode 100644 index 0000000000000..25b0e8cc01bec --- /dev/null +++ b/src/dispatch_trampoline.c @@ -0,0 +1,102 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "julia.h" +#include "julia_internal.h" + +// Process-global cache backing `jl_dispatch_trampolines` (the `Core.dispatch_trampolines` +// singleton). See the section comment below for the cache structure. + +// ---- dispatch-trampoline cache (@cfunction/@ccallable) ---- +// Maps (sigt, rt, specsig) -> jl_dispatch_trampoline_t, keyed on the resolution sig `sigt` = +// `Tuple{typeof(f), A...}` alone. Records sharing a `sigt` (differing in `rt` or `specsig`) are +// chained through `jl_dispatch_trampoline_t.next` and disambiguated by (rt, specsig). + +static jl_dispatch_trampoline_t *tramp_alloc_entry(jl_task_t *ct, jl_value_t *sigt, jl_value_t *rt, + int specsig) JL_CANSAFEPOINT +{ + jl_dispatch_trampoline_t *e = (jl_dispatch_trampoline_t*)jl_gc_alloc(ct->ptls, sizeof(jl_dispatch_trampoline_t), jl_dispatch_trampoline_type); + e->sigt = sigt; + e->rt = rt; + e->last_invokee = NULL; // unresolved + jl_atomic_store_relaxed(&e->fptr, (void*)NULL); + jl_atomic_store_relaxed(&e->last_world, (size_t)0); + jl_atomic_store_relaxed(&e->next, (jl_dispatch_trampoline_t*)NULL); + e->specsig = specsig ? 1 : 0; + return e; +} + +// Record key within a `sigt` bucket; the fields live packed inline on the record itself. +typedef struct { + jl_value_t *rt; + int specsig; +} tramp_key_t; + +// `rt` is compared by *type equality* (jl_types_equal), matching how the TypeMap matches +// `sigt`; `jl_egal` would split type-equal-but-not-egal return types into duplicate records. +// `specsig` is part of the key because uses_specsig depends on the emitting cgparams +// (prefer_specsig), not just (sigt, rt); a call site must get a record built for its own +// calling convention. +static int tramp_match(jl_value_t *rec, void *keyv) JL_CANSAFEPOINT +{ + jl_dispatch_trampoline_t *e = (jl_dispatch_trampoline_t*)rec; + tramp_key_t *k = (tramp_key_t*)keyv; + return (int)e->specsig == (k->specsig ? 1 : 0) + && (e->rt == k->rt || jl_types_equal(e->rt, k->rt)); +} + +// Lock-free lookup of the trampoline for (sigt, rt, specsig); NULL if absent. Safe to call +// with or without the writelock held. +static jl_dispatch_trampoline_t *tramp_map_lookup(jl_value_t *sigt, jl_value_t *rt, int specsig) JL_CANSAFEPOINT +{ + tramp_key_t key = { rt, specsig }; + return (jl_dispatch_trampoline_t*)jl_typemap_list_lookup(&jl_dispatch_trampolines->cache, + sigt, offsetof(jl_dispatch_trampoline_t, next), tramp_match, &key); +} + +// Insert `tr` into the `sigt`-keyed bucket. Caller holds the writelock and must have +// confirmed (under the lock) that (sigt, rt, specsig) is absent; `sigt`/`tr` must be kept +// rooted. +static void tramp_map_insert(jl_value_t *sigt, jl_dispatch_trampoline_t *tr) JL_CANSAFEPOINT +{ + jl_typemap_list_insert(&jl_dispatch_trampolines->cache, (jl_value_t*)jl_dispatch_trampolines, + sigt, (jl_value_t*)tr, offsetof(jl_dispatch_trampoline_t, next)); +} + +// Get (or create) the canonical @cfunction/@ccallable dispatch trampoline for +// (sigt, rt, specsig); call sites with the same key share one trampoline. Caller must root +// `sigt`/`rt`. +JL_DLLEXPORT jl_dispatch_trampoline_t *jl_get_dispatch_trampoline(jl_value_t *sigt, jl_value_t *rt, int specsig) JL_CANSAFEPOINT +{ + jl_dispatch_trampoline_t *e = NULL; + JL_GC_PUSH1(&e); + e = tramp_map_lookup(sigt, rt, specsig); // lock-free fast path + if (e == NULL) { + JL_LOCK(&jl_dispatch_trampolines->writelock); + e = tramp_map_lookup(sigt, rt, specsig); // re-check: another thread may have inserted + if (e == NULL) { + e = tramp_alloc_entry(jl_current_task, sigt, rt, specsig); + tramp_map_insert(sigt, e); + } + JL_UNLOCK(&jl_dispatch_trampolines->writelock); + } + JL_GC_POP(); + return e; +} + +// Insert `tr` into the running cache if its key is absent and return the canonical record; +// keep-first, like jl_specializations_get_or_insert (a losing `tr` is left standalone). Used +// by the load fixup to re-insert image-restored trampolines. +JL_DLLEXPORT jl_dispatch_trampoline_t *jl_insert_dispatch_trampoline(jl_dispatch_trampoline_t *tr) JL_CANSAFEPOINT +{ + jl_dispatch_trampoline_t *e = NULL; + JL_GC_PUSH2(&tr, &e); + JL_LOCK(&jl_dispatch_trampolines->writelock); + e = tramp_map_lookup(tr->sigt, tr->rt, tr->specsig); + if (e == NULL) { + tramp_map_insert(tr->sigt, tr); + e = tr; + } + JL_UNLOCK(&jl_dispatch_trampolines->writelock); + JL_GC_POP(); + return e; +} diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index b3bd44a2fe174..be8ea932fc61f 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -5,6 +5,8 @@ XX(abi_adapter_type, jl_datatype_t*) \ XX(abi_adapter_cache_type, jl_datatype_t*) \ XX(abi_adapters, jl_abi_adapter_cache_t*) \ + XX(dispatch_trampoline_cache_type, jl_datatype_t*) \ + XX(dispatch_trampolines, jl_dispatch_trampoline_cache_t*) \ XX(abioverride_type, jl_datatype_t*) \ XX(abstractarray_type, jl_unionall_t*) \ XX(abstractstring_type, jl_datatype_t*) \ @@ -129,6 +131,7 @@ XX(top_module, jl_module_t*) \ XX(trimfailure_type, jl_datatype_t*) \ XX(tuple_typename, jl_typename_t*) \ + XX(dispatch_trampoline_type, jl_datatype_t*) \ XX(tvar_type, jl_datatype_t*) \ XX(typeerror_type, jl_datatype_t*) \ XX(typeeq_type, jl_datatype_t*) \ diff --git a/src/jltypes.c b/src/jltypes.c index c27b9c40adfc3..e925cbc9fa12a 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3753,6 +3753,33 @@ void jl_init_types(void) JL_GC_DISABLED const static uint32_t abi_adapter_atomicfields[1] = { 0b100000 }; // next (field 5) jl_abi_adapter_type->name->atomicfields = abi_adapter_atomicfields; + // The latest-world dispatch trampoline record (see jl_dispatch_trampoline_t in julia.h), + // used by @cfunction/@ccallable. Mutable; fptr/last_world are atomic (the latest-world + // polling protocol). The trailing fields may be #undef until first resolution. + jl_dispatch_trampoline_type = + jl_new_datatype(jl_symbol("DispatchTrampoline"), core, jl_any_type, jl_emptysvec, + jl_perm_symsvec(7, + "sigt", // resolution sig (also the adapter ABI sig) + "rt", // declared return type (declrt for @cfunction) + "last_invokee", // resolved dispatch target: Union{CodeInstance, ABIAdapter} + "fptr", // ABI adapter + "last_world", + "specsig", // 1 if fptr is a specsig adapter + "next"), // next record in the `sigt` bucket chain (may be NULL) + jl_svec(7, + jl_any_type, // hash-consed Tuple type + jl_any_type, // hash-consed return type + jl_any_type, // Union{CodeInstance, ABIAdapter} (may be #undef) + pointer_void, // cached resolved adapter (atomic) + jl_ulong_type, // last_world (atomic) + jl_uint8_type, // specsig + jl_any_type), // next (atomic) + jl_emptysvec, + 0, 1, 2); + // fptr (field 3), last_world (field 4), next (field 6) are atomic (0-indexed). + const static uint32_t dispatch_trampoline_atomicfields[1] = { 0b1011000 }; + jl_dispatch_trampoline_type->name->atomicfields = dispatch_trampoline_atomicfields; + // Singleton ABIAdapterCache type: holds the `adapters` TypeMap root (jl_nothing / // TypeMapLevel / TypeMapEntry), replaced atomically on insert. The `jl_mutex_t writelock` // is a trailing hidden C field (not registered here), following jl_method_type's writelock. @@ -3765,6 +3792,17 @@ void jl_init_types(void) JL_GC_DISABLED const static uint32_t abi_adapter_cache_atomicfields[1] = { 0b1 }; // adapters jl_abi_adapter_cache_type->name->atomicfields = abi_adapter_cache_atomicfields; + // Singleton DispatchTrampolineCache type: sibling of ABIAdapterCache holding the `trampolines` + // TypeMap root, with its own trailing hidden `jl_mutex_t writelock` (separate lock). + jl_dispatch_trampoline_cache_type = + jl_new_datatype(jl_symbol("DispatchTrampolineCache"), core, jl_any_type, jl_emptysvec, + jl_perm_symsvec(1, "cache"), + jl_svec1(jl_any_type), // trampolines: TypeMap root (jl_nothing when empty) + jl_emptysvec, + 0, 1, 1); + const static uint32_t dispatch_trampoline_cache_atomicfields[1] = { 0b1 }; // trampolines + jl_dispatch_trampoline_cache_type->name->atomicfields = dispatch_trampoline_cache_atomicfields; + tv = jl_svec2(tvar("T"), tvar("N")); jl_abstractarray_type = (jl_unionall_t*) jl_new_abstracttype((jl_value_t*)jl_symbol("AbstractArray"), core, @@ -3827,6 +3865,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally jl_an_empty_memory_any = (jl_value_t*)jl_alloc_memory_any(0); // used internally jl_abi_adapters = jl_new_abi_adapter_cache(); // process-global ABI-adapter cache + jl_dispatch_trampolines = jl_new_dispatch_trampoline_cache(); // process-global @cfunction/@ccallable trampoline cache // finish initializing module Core core = jl_core_module; diff --git a/src/julia.h b/src/julia.h index 0ec087e680292..f0b05d978a745 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1075,7 +1075,7 @@ typedef struct _jl_abi_adapter_t { // Latest-world dispatch trampoline record (`jl_dispatch_trampolines->cache`), used by // @cfunction/@ccallable: holds a *signature*, not a fixed CodeInstance, and re-resolves the // target as the world advances (the inline call site polls `last_world` vs the world counter -// and re-resolves via jl_resolve_trampoline). Mutable; `fptr`/`last_world` are atomic. +// and re-resolves via jl_update_dispatch_trampoline). Mutable; `fptr`/`last_world` are atomic. typedef struct _jl_dispatch_trampoline_t { JL_DATA_TYPE jl_value_t *sigt; // key: resolution sig Tuple{typeof(f), A...} (for jl_get_specialization1) diff --git a/src/julia_internal.h b/src/julia_internal.h index 59e87440e2707..9ac6f4b1f97d4 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1735,14 +1735,14 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( jl_value_t *fobj, jl_datatype_t *result, htable_t *cache, jl_svec_t *fill, void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals) JL_CANSAFEPOINT; -JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data) JL_CANSAFEPOINT; // Returns the adapter fptr; `*invokee` (optional) receives the Union{CodeInstance, ABIAdapter} // it was derived from. JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst, jl_value_t **invokee) JL_CANSAFEPOINT; -// ABI-adapter cache (src/abi_adapter.c). The bucketed TypeMap +// ABI-adapter / dispatch-trampoline caches (src/abi_adapter.c, src/dispatch_trampoline.c). The bucketed TypeMap // bookkeeping lives in libjulia-internal; the JIT half (jl_get_abi_adapter) is in codegen. JL_DLLEXPORT jl_abi_adapter_cache_t *jl_new_abi_adapter_cache(void) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_dispatch_trampoline_cache_t *jl_new_dispatch_trampoline_cache(void) JL_CANSAFEPOINT; JL_DLLEXPORT int jl_abi_matches_invoke_api(jl_abi_t from_abi, jl_invoke_api_t api, jl_method_instance_t *mi, jl_value_t *rettype) JL_CANSAFEPOINT; // The target's own compiled entry point when it already satisfies `from_abi` (so no adapter @@ -1757,9 +1757,14 @@ JL_DLLEXPORT void *jl_lookup_abi_converter(jl_abi_t from_abi, jl_code_instance_t void **target, int *target_specsig, jl_callptr_t *invoke, jl_value_t **invokee) JL_CANSAFEPOINT; JL_DLLEXPORT jl_abi_adapter_t *jl_new_abi_adapter(jl_value_t *sigt, jl_value_t *rt, jl_code_instance_t *ci, int specsig, int is_opaque_closure, size_t nargs, void *fptr) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_dispatch_trampoline_t *jl_get_dispatch_trampoline(jl_value_t *sigt, jl_value_t *rt, int specsig) JL_CANSAFEPOINT; +JL_DLLEXPORT jl_dispatch_trampoline_t *jl_insert_dispatch_trampoline(jl_dispatch_trampoline_t *tr) JL_CANSAFEPOINT; // Insert the record if its key is absent and return the canonical record; keep-first, like // jl_specializations_get_or_insert. JL_DLLEXPORT jl_abi_adapter_t *jl_insert_abi_adapter(jl_abi_adapter_t *e) JL_CANSAFEPOINT; +// The @cfunction/@ccallable dispatch-trampoline resolver (runtime_ccall.c); the codegen +// poll calls it (as jlupdatetrampoline_func) when its cached world is stale. +JL_DLLEXPORT void *jl_update_dispatch_trampoline(jl_task_t *ct, jl_dispatch_trampoline_t *tr) JL_CANSAFEPOINT; // Special filenames used to refer to internal julia libraries diff --git a/src/staticdata.c b/src/staticdata.c index c10833dbe75a6..f1ba3f60b02fc 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -744,6 +744,19 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ jl_abi_adapter_cache_t *c = (jl_abi_adapter_cache_t*)v; record_field_change((jl_value_t**)&c->cache, jl_nothing); } + if (jl_typetagis(v, jl_dispatch_trampoline_cache_type)) { + // Likewise the trampoline cache: serialize it empty and re-insert each restored + // trampoline on load. + jl_dispatch_trampoline_cache_t *c = (jl_dispatch_trampoline_cache_t*)v; + record_field_change((jl_value_t**)&c->cache, jl_nothing); + } + if (jl_is_dispatch_trampoline(v)) { + // The `sigt` bucket chain is a process-local runtime cache, rebuilt on load + // (jl_insert_dispatch_trampoline); drop `next` so we neither relocate it nor drag an + // unreachable sibling trampoline into the image (mirrors ci->next handling). + jl_dispatch_trampoline_t *tr = (jl_dispatch_trampoline_t*)v; + record_field_change((jl_value_t**)&tr->next, NULL); + } if (jl_is_abi_adapter(v)) { // Like the cache itself, the adapter's `sigt` bucket chain is process-local and // rebuilt on load (jl_insert_abi_adapter), so drop `next`. @@ -1829,6 +1842,20 @@ static void jl_write_values(jl_serializer_state *s) JL_CANSAFEPOINT JL_GC_DISABL arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + BuiltinFunctionTag + builtin_id - 2)); // relocation target } } + else if (jl_is_dispatch_trampoline(v)) { + assert(f == s->s); + // `fptr` is a process-local address (a redundant cache of `last_invokee`'s + // fptr); null it and reset `last_world` to the unresolved sentinel. The first + // call after load re-validates `last_invokee` and restores `fptr` from it + // (jl_update_dispatch_trampoline). The adapter thunk itself is wired via the + // ABIAdapter record's own fptr_record entry, not the trampoline's. + jl_dispatch_trampoline_t *newtr = (jl_dispatch_trampoline_t*)&f->buf[reloc_offset]; + jl_atomic_store_relaxed(&newtr->fptr, NULL); + jl_atomic_store_relaxed(&newtr->last_world, 0); + // Re-insert this trampoline into the running cache on load, so a later + // @cfunction with the same (sigt, rt, specsig) shares it. + arraylist_push(&s->fixup_objs, (void*)reloc_offset); + } else if (jl_is_abi_adapter(v)) { assert(f == s->s); // The compiled adapter thunk lives in the image as an fvar. Null the record's @@ -1854,12 +1881,14 @@ static void jl_write_values(jl_serializer_state *s) JL_CANSAFEPOINT JL_GC_DISABL arraylist_push(&s->fixup_objs, (void*)reloc_offset); } } - else if (jl_typetagis(v, jl_abi_adapter_cache_type)) { + else if (jl_typetagis(v, jl_abi_adapter_cache_type) || jl_typetagis(v, jl_dispatch_trampoline_cache_type)) { assert(f == s->s); // The cache's `writelock` is a hidden trailing C field (not a datatype field); // pad the serialized object out to the full struct size so the restored lock // is zero-initialized (mirrors the jl_method_t writelock handling above). - write_padding(f, sizeof(jl_abi_adapter_cache_t) - tot); + size_t fullsz = jl_typetagis(v, jl_abi_adapter_cache_type) + ? sizeof(jl_abi_adapter_cache_t) : sizeof(jl_dispatch_trampoline_cache_t); + write_padding(f, fullsz - tot); } else if (jl_is_datatype(v)) { assert(f == s->s); @@ -4257,7 +4286,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, o->bits.in_image = 1; } arraylist_free(&cleanup_list); - // Restored ABI-adapter records, re-inserted into the running cache after + // Restored trampolines / ABI-adapter records, re-inserted into the running caches after // jl_update_all_fptrs below (see the comment there). arraylist_t reinsert_objs; arraylist_new(&reinsert_objs, 0); @@ -4268,7 +4297,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_ptr_1d_push(*internal_methods, obj); assert(s.incremental); } - else if (jl_is_abi_adapter(obj)) { + else if (jl_is_dispatch_trampoline(obj) || jl_is_abi_adapter(obj)) { // Deferred (see reinsert_objs above): re-inserted after jl_update_all_fptrs. arraylist_push(&reinsert_objs, (void*)obj); } @@ -4377,13 +4406,18 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_update_all_fptrs(&s, image); // fptr relocs and registration s.s = NULL; - // Re-insert the restored ABI-adapter records into the running cache, so matching - // consumers share them (and reuse the compiled adapters without a JIT). This publishes - // the records to lock-free readers, so it must happen only now, after + // Re-insert the restored trampolines and ABI-adapter records into the running caches, so + // matching consumers share them (and reuse the compiled adapters without a JIT). This + // publishes the records to lock-free readers, so it must happen only now, after // jl_update_all_fptrs wired each record's `fptr`: a reader must never observe a NULL // `fptr`. - for (size_t i = 0; i < reinsert_objs.len; i++) - jl_insert_abi_adapter((jl_abi_adapter_t*)reinsert_objs.items[i]); + for (size_t i = 0; i < reinsert_objs.len; i++) { + jl_value_t *obj = (jl_value_t*)reinsert_objs.items[i]; + if (jl_is_dispatch_trampoline(obj)) + jl_insert_dispatch_trampoline((jl_dispatch_trampoline_t*)obj); + else + jl_insert_abi_adapter((jl_abi_adapter_t*)obj); + } arraylist_free(&reinsert_objs); ios_close(&fptr_record); From 1a9e8641a7f21deef3a4282d2d91c2823ea149be Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 1 Jul 2026 13:38:19 -0400 Subject: [PATCH 4/5] codegen: move @cfunction/@ccallable to ABIAdapter/DispatchTrampoline; delete cfuncdata Reroute @cfunction/@ccallable off the per-call-site cfuncdata global onto the interned DispatchTrampoline + ABIAdapter caches. emit_abi_call now embeds the interned trampoline and polls last_world/fptr, falling back to the runtime resolver jl_resolve_trampoline (which restores fptr from the trampoline's last_invoked, JITing/interning an ABIAdapter only when needed). The AOT path (generate_cfunc_thunks/emit_trampoline_adapter) materializes each trampoline's adapter record and, outside --trim, eagerly enqueues the dynamic-dispatch ("unspecialized") adapter so a codegen-free interpreter run of a non-trimmed image can still dispatch. The obsolete cfuncdata_t struct and jl_get_abi_converter are removed. Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Claude Fable 5 --- src/aotcompile.cpp | 8 +- src/codegen.cpp | 73 +++++++++------- src/jitlayers.h | 13 ++- src/jl_exported_funcs.inc | 1 + src/julia_internal.h | 1 + src/runtime_ccall.c | 155 +++++++++++++++++++------------- src/staticdata.c | 7 ++ test/ccall.jl | 180 ++++++++++++++++++++++++++++++++++++++ test/precompile.jl | 85 ++++++++++++++++++ 9 files changed, 420 insertions(+), 103 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 3b6b21190d72b..49e0f35251c9c 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -598,7 +598,7 @@ static Function *aot_abi_converter(jl_codegen_output_t &out, jl_abi_t from_abi, return F; } -// AOT counterpart of jl_resolve_trampoline: resolve the target for `tr` at the build world, +// AOT counterpart of jl_update_dispatch_trampoline: resolve the target for `tr` at the build world, // emit its ABI adapter, and record the materialized ABIAdapter record in // `out.tramp_invokees` / `out.adapter_funcs`. When the target's compiled ABI already // satisfies the declared C ABI, `tramp_invokees` maps to the bare `CodeInstance` instead and @@ -664,7 +664,7 @@ static void emit_trampoline_adapter(jl_codegen_output_t &out, jl_dispatch_trampo // fvar on load) and map the trampoline to it. The record reaches the image via the // serializer's `last_invokee` override (and ext_foreign_code); the first post-load call // restores `fptr` straight from it. - jl_abi_adapter_t *rec = jl_new_abi_adapter_record(sigt, declrt, codeinst, + jl_abi_adapter_t *rec = jl_new_abi_adapter(sigt, declrt, codeinst, from_abi.specsig, from_abi.is_opaque_closure, from_abi.nargs, /*fptr*/nullptr); // Nothing else roots the record until it is reported to ext_foreign_code. JL_GC_PUSH1(&rec); @@ -677,7 +677,7 @@ static void emit_trampoline_adapter(jl_codegen_output_t &out, jl_dispatch_trampo // Emit the adapter for each @cfunction/@ccallable dispatch trampoline (kind=STD, so the // adapter sig is the call sig as-is). Replaces the old cfuncdata-array fill; the serializer // wires the adapter into the image trampoline's `last_invokee` (jl_get_trampoline_invokee). -// Call sites sharing one interned trampoline (same sigt/rt/specsig) are deduplicated here, so +// Call sites sharing one trampoline (same sigt/rt/specsig) are deduplicated here, so // each trampoline yields exactly one adapter record and one unspecialized record. static void generate_cfunc_thunks(jl_codegen_output_t &out) JL_CANSAFEPOINT { @@ -702,7 +702,7 @@ static void generate_cfunc_thunks(jl_codegen_output_t &out) JL_CANSAFEPOINT // jl_create_native_impl). Under --trim, dynamic dispatch is unavailable, so skip it. if (!jl_options.trim) { Function *uf = aot_abi_converter(out, cfunc.abi, nullptr, nullptr, nullptr, false); - jl_abi_adapter_t *urec = jl_new_abi_adapter_record(cfunc.abi.sigt, cfunc.abi.rt, + jl_abi_adapter_t *urec = jl_new_abi_adapter(cfunc.abi.sigt, cfunc.abi.rt, /*ci*/nullptr, cfunc.abi.specsig, cfunc.abi.is_opaque_closure, cfunc.abi.nargs, /*fptr*/nullptr); JL_GC_PUSH1(&urec); diff --git a/src/codegen.cpp b/src/codegen.cpp index eec2ecf3ef5ae..6c2d129b7796b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1406,11 +1406,12 @@ static const auto jlgetcfunctiontrampoline_func = new JuliaFunction<>{ Attributes(C, {Attribute::NonNull}), {}); }, }; -static const auto jlgetabiconverter_func = new JuliaFunction{ - XSTR(jl_get_abi_converter), - [](LLVMContext &C, Type *T_size) { +static const auto jlupdatetrampoline_func = new JuliaFunction<>{ + XSTR(jl_update_dispatch_trampoline), + [](LLVMContext &C) { Type *T_ptr = getPointerTy(C); - return FunctionType::get(T_ptr, {T_ptr, T_ptr}, false); + Type *T_prjlvalue = JuliaType::get_prjlvalue_ty(C); + return FunctionType::get(T_ptr, {T_ptr, T_prjlvalue}, false); }, nullptr, }; @@ -7560,8 +7561,11 @@ std::string emit_abi_converter(jl_codegen_output_t &out, jl_abi_t from_abi, jl_c bool needsparams = false; bool target_is_opaque_closure = false; jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + // Unique per emission (like emit_abi_dispatcher / emit_abi_constreturn): each ABIAdapter record + // gets its own distinct adapter Function, so the fvar table has no duplicates and `fptr_record` + // stays 1:1 (fvar id -> one record) -- mirroring how each CodeInstance gets its own Function. std::string gf_thunk_name = get_function_name(from_abi.specsig, needsparams, name_from_method_instance(mi), out.TargetTriple); - gf_thunk_name += "_gfthunk"; + raw_string_ostream(gf_thunk_name) << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1) << "_gfthunk"; if (target_specsig) { jl_value_t *abi = get_ci_abi(codeinst); jl_returninfo_t targetspec = get_specsig_function(out, M, target, "", abi, codeinst->rettype, target_is_opaque_closure); @@ -7659,41 +7663,44 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ bool needsparams = false; bool is_opaque_closure = false; bool specsig = uses_specsig(sigt, needsparams, declrt, ctx.params->prefer_specsig); - PointerType *T_ptr = ctx.types().T_ptr; Type *T_size = ctx.types().T_size; - Constant *Vnull = ConstantPointerNull::get(T_ptr); Module *M = jl_Module; - ArrayType *T_cfuncdata = ArrayType::get(T_ptr, 8); - size_t flags = specsig; - GlobalVariable *cfuncdata = new GlobalVariable(*M, T_cfuncdata, false, - GlobalVariable::PrivateLinkage, - ConstantArray::get(T_cfuncdata, { - Vnull, - Vnull, - Vnull, - Vnull, - Vnull, - literal_pointer_val_slot(ctx.emission_context, declrt), - literal_pointer_val_slot(ctx.emission_context, sigt), - literal_static_pointer_val((void*)flags, T_ptr)})); - Value *last_world_p = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, cfuncdata, 1); - LoadInst *last_world_v = ctx.builder.CreateAlignedLoad(T_size, last_world_p, ctx.types().alignof_ptr); + // Get the canonical @cfunction/@ccallable dispatch trampoline: a GC-tracked + // jl_dispatch_trampoline_t that re-resolves the target at the latest world, shared + // among call sites with the same (sigt, rt, specsig). Its fptr is filled lazily by + // jl_update_dispatch_trampoline (JIT) or compiled into the image by generate_cfunc_thunks + // (AOT). + jl_dispatch_trampoline_t *tr = NULL; + JL_GC_PUSH1(&tr); + tr = jl_get_dispatch_trampoline(sigt, declrt, specsig); + jl_temporary_root(ctx, (jl_value_t*)tr); // keep alive through codegen + serialization + JL_GC_POP(); + jl_abi_t cfuncabi = {sigt, declrt, nargs, specsig, is_opaque_closure}; + ctx.emission_context.cfuncs.push_back({cfuncabi, tr}); + Value *tramp_box = boxed(ctx, mark_julia_const(ctx, (jl_value_t*)tr)); + Value *tramp_d = decay_derived(ctx, tramp_box); + // Poll the record last_world -> fptr -> counter, all acquire (the release stores in + // jl_update_dispatch_trampoline pair with these). Raw atomic loads at the field offsets: the + // record is a codegen constant, so a getfield could be const-folded -- the poll must + // re-read. Install the polled world for the call; the cfunction wrapper restores the + // caller's world around emit_abi_call. + Value *lw_p = emit_ptrgep(ctx, tramp_d, offsetof(jl_dispatch_trampoline_t, last_world)); + LoadInst *last_world_v = ctx.builder.CreateAlignedLoad(T_size, lw_p, ctx.types().alignof_ptr); last_world_v->setOrdering(AtomicOrdering::Acquire); - LoadInst *callee = ctx.builder.CreateAlignedLoad(T_ptr, cfuncdata, ctx.types().alignof_ptr); - callee->setOrdering(AtomicOrdering::Acquire); - LoadInst *world_v = ctx.builder.CreateAlignedLoad(ctx.types().T_size, + Value *fptr_p = emit_ptrgep(ctx, tramp_d, offsetof(jl_dispatch_trampoline_t, fptr)); + LoadInst *cached_fptr = ctx.builder.CreateAlignedLoad(ctx.types().T_ptr, fptr_p, ctx.types().alignof_ptr); + cached_fptr->setOrdering(AtomicOrdering::Acquire); + LoadInst *world_v = ctx.builder.CreateAlignedLoad(T_size, prepare_global_in(M, jlgetworld_global), ctx.types().alignof_ptr); - world_v->setOrdering(AtomicOrdering::Monotonic); + world_v->setOrdering(AtomicOrdering::Acquire); ctx.builder.CreateStore(world_v, world_age_field); Value *age_not_ok = ctx.builder.CreateICmpNE(last_world_v, world_v); - Value *target = emit_guarded_test(ctx, age_not_ok, callee, [&] () { - Function *getcaller = prepare_call(jlgetabiconverter_func); - CallInst *cw = ctx.builder.CreateCall(getcaller, {get_current_task(ctx), cfuncdata}); - cw->setAttributes(getcaller->getAttributes()); + Value *target = emit_guarded_test(ctx, age_not_ok, cached_fptr, [&] () { + Function *resolver = prepare_call(jlupdatetrampoline_func); + CallInst *cw = ctx.builder.CreateCall(resolver, {get_current_task(ctx), tramp_box}); + cw->setAttributes(resolver->getAttributes()); return cw; }); - jl_abi_t cfuncabi = {sigt, declrt, nargs, specsig, is_opaque_closure}; - ctx.emission_context.cfuncs.push_back({cfuncabi, cfuncdata}); if (specsig) { // TODO: could we force this to guarantee passing a box for `f` here (since we // know we had it here) and on the receiver end (emit_abi_converter / @@ -10594,7 +10601,7 @@ static void init_jit_functions(void) add_named_global(jlunlockvalue_func, &jl_unlock_value); add_named_global(jllockfield_func, &jl_lock_field); add_named_global(jlunlockfield_func, &jl_unlock_field); - add_named_global(jlgetabiconverter_func, &jl_get_abi_converter); + add_named_global(jlupdatetrampoline_func, &jl_update_dispatch_trampoline); jl_get_pgcstack_func_t get_pgcstack; jl_pgcstack_key_t pgcstack_key; diff --git a/src/jitlayers.h b/src/jitlayers.h index a7a3ddb30f3bb..05f87978fa7d7 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -375,10 +375,12 @@ struct jl_codegen_call_target_t { // neither = unused }; -// reification of a call to jl_jit_abi_convert, so that it isn't necessary to parse the Modules to recover this info +// reification of a @cfunction/@ccallable construction: the interned dispatch trampoline +// whose adapter generate_cfunc_thunks compiles into the image (replacing the old per-call-site +// cfuncdata global), plus the caller ABI it was emitted for. struct cfunc_decl_t { jl_abi_t abi; - llvm::GlobalVariable *cfuncdata; + jl_dispatch_trampoline_t *tramp; }; std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, @@ -465,7 +467,12 @@ class jl_codegen_output_t { // ABIAdapter records compiled for @cfunction/@ccallable serialization: each entry pairs a // materialized ABIAdapter record with the Function emitted for its resolved target, so the // serialized record's `fptr` is wired to that Function on load without a JIT. - SmallVector, 0> abi_adapter_records; + SmallVector, 0> adapter_funcs; + // The AOT-resolved target per @cfunction/@ccallable trampoline: Union{CodeInstance, + // ABIAdapter} (a bare CodeInstance when the declared ABI matches its specptr). Consulted + // by the serializer (jl_get_trampoline_invokee) when writing a trampoline's + // `last_invokee`. + DenseMap tramp_invokees; std::map global_targets; jl_array_t *temporary_roots = nullptr; SmallSet temporary_roots_set; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index bebf6eb8389e4..d9553d4cedc92 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -553,6 +553,7 @@ YY(jl_emit_native) \ YY(jl_get_function_id) \ YY(jl_get_adapter_id) \ + YY(jl_get_trampoline_invokee) \ YY(jl_struct_to_llvm) \ YY(jl_type_to_llvm) \ YY(jl_getUnwindInfo) \ diff --git a/src/julia_internal.h b/src/julia_internal.h index 9ac6f4b1f97d4..a9d3ebe6a1819 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -2267,6 +2267,7 @@ JL_DLLIMPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncod int32_t *func_idx, int32_t *specfunc_idx); JL_DLLIMPORT void jl_get_adapter_id(void *native_code, jl_abi_adapter_t *rec, int32_t *adapter_idx); +JL_DLLIMPORT jl_value_t *jl_get_trampoline_invokee(void *native_code, jl_dispatch_trampoline_t *tr); JL_DLLIMPORT void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_code_instance_t **linfos, size_t n); JL_DLLIMPORT void jl_get_llvm_cis(void *native_code, size_t *num_els, diff --git a/src/runtime_ccall.c b/src/runtime_ccall.c index ff55e242d745f..55917b90ba539 100644 --- a/src/runtime_ccall.c +++ b/src/runtime_ccall.c @@ -336,17 +336,6 @@ jl_value_t *jl_get_cfunction_trampoline( } JL_GCC_IGNORE_STOP -struct cfuncdata_t { - _Atomic(void *) fptr; - _Atomic(size_t) last_world; - jl_code_instance_t** plast_codeinst; - jl_code_instance_t* last_codeinst; - void *unspecialized; - jl_value_t *const *const declrt; - jl_value_t *const *const sigt; - size_t flags; -}; - static inline const char *name_from_method_instance(jl_method_instance_t *mi) JL_NOTSAFEPOINT { assert(jl_is_method_instance(mi)); @@ -369,86 +358,126 @@ static jl_mutex_t cfun_lock; // dispatch site observes last_world == jl_world_counter then the loaded // fptr is consistent with both of them, meaning it was published for // exactly that world. +// Corollary: a published (non-NULL) fptr must never be rewritten to NULL -- +// a lock-free poller may pair a stale last_world load with a current fptr +// load and call whatever it reads. fptr only ever moves valid -> valid. +// The resolved CodeInstance behind a trampoline's `last_invokee` (Union{CodeInstance, +// ABIAdapter}): the ABIAdapter's target CI, the bare CodeInstance itself, or NULL (unresolved / +// no method). +static jl_code_instance_t *invokee_ci(jl_value_t *last_invokee) JL_NOTSAFEPOINT +{ + if (last_invokee == NULL) + return NULL; + if (jl_is_abi_adapter(last_invokee)) + return ((jl_abi_adapter_t*)last_invokee)->ci; + return (jl_code_instance_t*)last_invokee; // a bare CodeInstance (declared ABI matched its specptr) +} + +// Update a @cfunction/@ccallable dispatch trampoline to the current world (JIT or +// re-validate its ABI adapter), publishing the resulting adapter in `tr->fptr`. Called by +// the inline call-site poll (emit_abi_call) when its cached `last_world` is stale. JL_DLLEXPORT -void *jl_get_abi_converter(jl_task_t *ct, void *data) +void *jl_update_dispatch_trampoline(jl_task_t *ct, jl_dispatch_trampoline_t *tr) JL_CANSAFEPOINT { - struct cfuncdata_t *cfuncdata = (struct cfuncdata_t*)data; - jl_value_t *sigt = *cfuncdata->sigt; + jl_value_t *sigt = tr->sigt; JL_GC_PROMISE_ROOTED(sigt); - jl_value_t *declrt = *cfuncdata->declrt; - JL_GC_PROMISE_ROOTED(declrt); - int specsig = cfuncdata->flags & 1; + jl_value_t *rt = tr->rt; + JL_GC_PROMISE_ROOTED(rt); size_t nargs = jl_nparams(sigt); + jl_abi_t from_abi = { sigt, rt, nargs, tr->specsig, /*is_opaque_closure*/0 }; jl_value_t *mi; jl_code_instance_t *codeinst; size_t world; - // check first, while behind this lock, of the validity of the current contents of this cfunc thunk JL_LOCK(&cfun_lock); do { - size_t last_world_v = jl_atomic_load_relaxed(&cfuncdata->last_world); - void *f = jl_atomic_load_relaxed(&cfuncdata->fptr); - jl_code_instance_t *last_ci = cfuncdata->plast_codeinst ? *cfuncdata->plast_codeinst : NULL; - JL_GC_PROMISE_ROOTED(last_ci); // cached CI is retained by the MI cache or by an image literal root slot + size_t last_world_v = jl_atomic_load_relaxed(&tr->last_world); + void *f = jl_atomic_load_relaxed(&tr->fptr); + jl_value_t *last_invokee = tr->last_invokee; + JL_GC_PROMISE_ROOTED(last_invokee); // retained by the adapter/MI cache or an image root + jl_code_instance_t *last_ci = invokee_ci(last_invokee); world = jl_atomic_load_acquire(&jl_world_counter); ct->world_age = world; - if (world == last_world_v) { + if (f != NULL && world == last_world_v) { JL_UNLOCK(&cfun_lock); return f; } mi = jl_get_specialization1((jl_tupletype_t*)sigt, world); - if (f != NULL) { - if (last_ci == NULL) { - if (mi == jl_nothing) { - jl_atomic_store_release(&cfuncdata->last_world, world); - JL_UNLOCK(&cfun_lock); - return f; + // Re-validate without inferring: if `last_invokee` still matches the dispatch result + // for this world (same mi, or both indicate "no method") and its CodeInstance is + // still valid at `world`, restore `fptr` straight from it -- an ABIAdapter carries + // its compiled thunk's `fptr`; a bare CodeInstance means the declared ABI matched its + // specptr, so re-derive that. + if (last_invokee != NULL) { + int still_valid = last_ci == NULL + ? (mi == jl_nothing) + : ((jl_value_t*)jl_get_ci_mi(last_ci) == mi && + jl_atomic_load_relaxed(&last_ci->max_world) >= world); + if (still_valid) { + void *nf = f; + if (nf == NULL) { // first call after load: restore fptr directly from `last_invokee` + if (jl_is_abi_adapter(last_invokee)) { + nf = ((jl_abi_adapter_t*)last_invokee)->fptr; + } + else { + nf = jl_abi_matching_specptr(from_abi, last_ci); + } } - } - else { - if ((jl_value_t*)jl_get_ci_mi(last_ci) == mi && jl_atomic_load_relaxed(&last_ci->max_world) >= world) { // same dispatch and source - jl_atomic_store_release(&cfuncdata->last_world, world); + if (nf != NULL) { + jl_atomic_store_release(&tr->fptr, nf); + jl_atomic_store_release(&tr->last_world, world); JL_UNLOCK(&cfun_lock); - return f; + return nf; } } } JL_UNLOCK(&cfun_lock); - // next, try to figure out what the target should look like (outside of the lock since this is very slow) + // slow: infer the target outside the lock (this is very slow) codeinst = mi != jl_nothing ? jl_type_infer((jl_method_instance_t*)mi, world, SOURCE_MODE_ABI, jl_options.trim) : NULL; - // relock for the remainder of the function + // Compile the target now, so its specptr is available to the specptr shortcut + // (jl_abi_matching_specptr / jl_lookup_abi_adapter never compile). + if (codeinst != NULL) + jl_compile_codeinst(codeinst); JL_LOCK(&cfun_lock); - } while (jl_atomic_load_acquire(&jl_world_counter) != world); // restart entirely, since jl_world_counter changed thus jl_get_specialization1 might have changed - // double-check if the values were set on another thread - size_t last_world_v = jl_atomic_load_relaxed(&cfuncdata->last_world); - void *f = jl_atomic_load_relaxed(&cfuncdata->fptr); - if (world == last_world_v) { + } while (jl_atomic_load_acquire(&jl_world_counter) != world); // restart if the world moved under us + // double-check another thread didn't already install for this world + size_t lwv = jl_atomic_load_relaxed(&tr->last_world); + void *f = jl_atomic_load_relaxed(&tr->fptr); + if (f != NULL && world == lwv) { JL_UNLOCK(&cfun_lock); - return f; // another thread fixed this up while we were away + return f; } - int is_opaque_closure = 0; - jl_abi_t from_abi = { sigt, declrt, nargs, specsig, is_opaque_closure }; - if (codeinst == NULL) { - // Generate an adapter to a dynamic dispatch - if (cfuncdata->unspecialized == NULL) - cfuncdata->unspecialized = jl_jit_abi_converter(ct, from_abi, NULL, NULL); - - f = cfuncdata->unspecialized; - } else { + // If the world advanced but the dispatch target is unchanged from the one `fptr` was + // built for, the existing adapter is still correct -- reuse it (no re-JIT). + if (f != NULL && codeinst == invokee_ci(tr->last_invokee)) { + jl_atomic_store_release(&tr->last_world, world); + JL_UNLOCK(&cfun_lock); + return f; + } + // @cfunction warns when the declared C return type cannot match the resolved target's + // rettype (skip must-not-return targets, occasionally required by the C API for error + // callbacks even though memory errors are then likely). + if (codeinst != NULL) { jl_value_t *astrt = codeinst->rettype; if (astrt != (jl_value_t*)jl_bottom_type && - jl_type_intersection(astrt, declrt) == jl_bottom_type) { - // Do not warn if the function never returns since it is - // occasionally required by the C API (typically error callbacks) - // even though we're likely to encounter memory errors in that case - jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance((jl_method_instance_t*)mi)); - } - f = jl_jit_abi_converter(ct, from_abi, codeinst, NULL); + jl_type_intersection(astrt, rt) == jl_bottom_type) + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", + name_from_method_instance((jl_method_instance_t*)mi)); } - - cfuncdata->plast_codeinst = &cfuncdata->last_codeinst; - cfuncdata->last_codeinst = codeinst; - jl_atomic_store_release(&cfuncdata->fptr, f); - jl_atomic_store_release(&cfuncdata->last_world, world); + // Resolve the adapter and record `last_invokee` as a Union{CodeInstance, ABIAdapter}: a bare + // CodeInstance when the declared ABI already matches its specptr (no thunk needed), else the + // cached ABIAdapter record (a NULL codeinst yields the shared dynamic-dispatch adapter). + // A later re-validation restores `fptr` straight from it. + jl_value_t *new_invokee = NULL; + f = jl_jit_abi_converter(ct, from_abi, codeinst, &new_invokee); + // published before fptr/last_world; roots the target + jl_gc_write(tr, tr->last_invokee, jl_value_t, new_invokee); + // Publish `fptr` with *release*: the call-site poll acquire-loads it, and this release + // pairs with that acquire so observing this fptr also makes our prior acquire-load of + // `jl_world_counter` visible -- closing the window where a lock-free reader could pair a + // newer fptr with a stale counter. `last_world` (release, stored after fptr) gives the + // reader the lower bound. + jl_atomic_store_release(&tr->fptr, f); + jl_atomic_store_release(&tr->last_world, world); JL_UNLOCK(&cfun_lock); return f; } diff --git a/src/staticdata.c b/src/staticdata.c index f1ba3f60b02fc..22331c7a8bfe9 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -756,6 +756,13 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ // unreachable sibling trampoline into the image (mirrors ci->next handling). jl_dispatch_trampoline_t *tr = (jl_dispatch_trampoline_t*)v; record_field_change((jl_value_t**)&tr->next, NULL); + // Serialize `last_invokee` as the target this emission compiled (from the native-code + // side table), never as the live runtime state: a JIT-resolved record's `fptr` is a + // process address that cannot be wired on load. With no entry (or no native code) the + // trampoline serializes as unresolved, and the first post-load call resolves fresh. + jl_value_t *invokee = native_functions ? + jl_get_trampoline_invokee(native_functions, tr) : NULL; + record_field_change((jl_value_t**)&tr->last_invokee, invokee); } if (jl_is_abi_adapter(v)) { // Like the cache itself, the adapter's `sigt` bucket chain is process-local and diff --git a/test/ccall.jl b/test/ccall.jl index 4fc8a7158104f..67716821eebff 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -2120,3 +2120,183 @@ const sym = :ZSTD_versionString get_zstd_version() = prefix * unsafe_string(ccall((sym, libzstd), Cstring, ())) @test startswith(get_zstd_version(), "Zstd") end + +# ABI-adapter cache: cfunctions run end-to-end through the `jl_abi_adapters` cache (wired +# via jl_jit_abi_converter), and the runtime cache stays bounded across repeated calls. +# Signatures whose declared ABI matches the method's specsig exactly short-circuit to +# `specptr` without producing an adapter, so the cache may legitimately stay empty for them. +# The cases below cover the adapter branches reachable from `@cfunction`: specsig (widen/narrow), +# const-return, and the boxed args ABI, plus the unresolved (codeinst == NULL) dispatcher (both a +# missing method that throws and an abstract `Any` argument that dispatches at call time). The +# JL_INVOKE_SPARAM branch is not exercisable here: `@cfunction` resolves its target via +# jl_get_specialization1, which for a concrete signature returns a concrete specialization (whose +# static parameters are fixed -> specsig) and for an abstract signature (e.g. an `Any` argument) +# returns nothing -> the codeinst == NULL dispatcher. The generic `jl_fptr_sparam` entry is never +# selected; it is only reachable when a generic CodeInstance is handed to the trampoline directly. +# Targets for the ABI-adapter cache test (must be globals: `@cfunction` over a bare name +# needs a compile-time-known function). `abi_match_target` matches its declared C ABI; +# `abi_narrow_target` deliberately infers to `Any` (via compilerbarrier). +abi_match_target(x::Cint) = x + Cint(1) # Cint -> Cint +abi_widen_target(x::Cint) = x # well-inferred Cint +abi_narrow_target(x::Cint) = Core.compilerbarrier(:type, x + Cint(1)) # inferred ::Any +abi_const_target(x::Cint) = Cint(42) # const-return (JL_INVOKE_CONST) +abi_args_target(x::Cint...) = length(x) % Cint # vararg (JL_INVOKE_ARGS) +abi_unresolved_target(x::Float64) = x # no method for (Cint,) + +@testset "abi adapter cache" begin + adapter_count() = ccall(:jl_typemap_count, Csize_t, (Any,), + getfield(Core.abi_adapters, :cache)) + + # (1) Matching ABI (Cint -> Cint, both specsig/isbits): jl_jit_abi_converter short-circuits + # to the method's specptr, so no adapter is emitted and the cache is untouched. + let cf = @cfunction(abi_match_target, Cint, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test ccall(fptr, Cint, (Cint,), Cint(3)) == Cint(4) + @test adapter_count() == n0 + end + end + + # (2) Widening: an `Any` cfunction return over a well-inferred (Cint) CodeInstance forces + # a boxing adapter -- no specptr shortcut, so it is emitted and cached on first call, and + # re-resolving the same trampoline reuses `tr->fptr` without growing the cache. + let cf = @cfunction(abi_widen_target, Any, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test ccall(fptr, Any, (Cint,), Cint(5)) === Cint(5) + n1 = adapter_count() + @test n1 > n0 + @test ccall(fptr, Any, (Cint,), Cint(6)) === Cint(6) + @test adapter_count() == n1 + end + end + + # (3) Narrowing: a `Cint` cfunction return over an `Any`-returning CodeInstance forces an + # unbox/convert adapter (the boxed result is narrowed to the declared C type). + let cf = @cfunction(abi_narrow_target, Cint, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test ccall(fptr, Cint, (Cint,), Cint(7)) == Cint(8) + @test adapter_count() > n0 + end + end + + # (4) Const-ABI: a const-return CodeInstance has no specptr to shortcut to, so the adapter + # is emitted via the const-return path (JL_INVOKE_CONST). + let cf = @cfunction(abi_const_target, Cint, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test ccall(fptr, Cint, (Cint,), Cint(3)) == Cint(42) + @test adapter_count() > n0 + end + end + + # (5) Args-ABI: a vararg method is compiled to the boxed `jl_fptr_args` ABI, which never + # matches the declared C signature, so the adapter routes through the specptr (JL_INVOKE_ARGS). + let cf = @cfunction(abi_args_target, Cint, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test ccall(fptr, Cint, (Cint,), Cint(9)) == Cint(1) + @test adapter_count() > n0 + end + end + + # (6) Unresolved target: no method matches the declared signature, so the trampoline resolves + # to the shared dynamic-dispatch adapter (codeinst == NULL), which throws on the missing method. + let cf = @cfunction(abi_unresolved_target, Cint, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test_throws MethodError ccall(fptr, Cint, (Cint,), Cint(3)) + @test adapter_count() > n0 + end + end + + # (7) Abstract argument: an `Any`-typed argument makes the signature non-concrete, so + # jl_get_specialization1 returns nothing and the same dynamic-dispatch adapter (codeinst == + # NULL) is used -- here the dispatch succeeds at call time rather than throwing. + let cf = @cfunction(abi_match_target, Any, (Any,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + n0 = adapter_count() + @test ccall(fptr, Any, (Any,), Cint(3)) === Cint(4) + @test adapter_count() > n0 + end + end +end + +# Regression test for the independent (ci == NULL, owned by no trampoline) ABI adapters that +# generate_cfunc_thunks eagerly emits for each @cfunction/@ccallable site outside --trim (the +# "unspecialized" dynamic-dispatch adapter). These are rooted for serialization only via +# `ext_foreign_code`, so a full / non-worklist system image must still bake them -- otherwise a +# codegen-free load cannot dispatch these calls without JITing an adapter (#61949). A freshly +# started base image must therefore expose a non-empty `Core.abi_adapters` cache populated purely +# from its own @cfunction/@ccallable sites. Checked in a subprocess: the cfunctions created by the +# "abi adapter cache" testset above populate the cache in *this* process, which would otherwise +# mask a regression back to an empty serialized cache. +@testset "eager-unspecialized adapters serialized into the sysimage" begin + countexpr = "print(ccall(:jl_typemap_count, Csize_t, (Any,), getfield(Core.abi_adapters, :cache)))" + out = readchomp(`$(Base.julia_cmd()) --startup-file=no -e $countexpr`) + @test parse(Int, out) > 0 +end + +# DispatchTrampoline cache: `jl_get_dispatch_trampoline` returns the canonical dispatch +# record per (sigt, rt, specsig). Records sharing a `sigt` (differing in `rt` or `specsig`) +# chain within one TypeMap entry; a distinct `sigt` gets its own entry. `specsig` is part of +# the key because `uses_specsig` depends on the emitting cgparams (prefer_specsig), so call +# sites emitted under different params must not share a record built for the other +# convention. +@testset "dispatch trampoline cache" begin + tramp_target(x::Int) = x + entries() = ccall(:jl_typemap_count, Csize_t, (Any,), + getfield(Core.dispatch_trampolines, :cache)) + get_tramp(sigt, rt, specsig=true) = ccall(:jl_get_dispatch_trampoline, Any, + (Any, Any, Cint), sigt, rt, Cint(specsig))::Core.DispatchTrampoline + sigt = Tuple{typeof(tramp_target), Int} + n0 = entries() + a = get_tramp(sigt, Int) + b = get_tramp(sigt, Int) + @test a === b # canonical: same (sigt, rt, specsig) -> same record + @test getfield(a, :sigt) === sigt + @test getfield(a, :rt) === Int + @test entries() == n0 + 1 + c = get_tramp(sigt, Float64) # same sigt, different rt -> distinct record, same entry + @test c !== a + @test entries() == n0 + 1 + s = get_tramp(sigt, Int, false) # same (sigt, rt), different specsig -> distinct record, same entry + @test s !== a + @test getfield(s, :specsig) === UInt8(0) + @test entries() == n0 + 1 + @test get_tramp(sigt, Int) === a # original record still canonical for specsig=true + d = get_tramp(Tuple{typeof(tramp_target), Float64}, Int) # different sigt -> new entry + @test d !== a + @test entries() == n0 + 2 +end + +# A cache record without a compiled thunk (fptr == NULL) must be repaired, not served: e.g. a +# runtime-created ABIAdapter record stored in serialized user data has no fvar slot, so it +# would be restored with a NULL fptr. Pre-poison the dispatcher key for a signature, then +# resolve a @cfunction for it: the JIT must fill the record in place rather than hand out the +# NULL fptr or shadow the key with a duplicate. +poison_target(x::Float64) = x # no method for (Cint,) -> the resolver uses the dispatcher +@testset "NULL-fptr adapter records are repaired" begin + sigt = Tuple{typeof(poison_target), Cint} + mkrec(ss) = ccall(:jl_new_abi_adapter, Any, + (Any, Any, Ptr{Cvoid}, Cint, Cint, Csize_t, Ptr{Cvoid}), + sigt, Any, C_NULL, Cint(ss), Cint(0), Csize_t(2), C_NULL)::Core.ABIAdapter + recs = [mkrec(ss) for ss in (0, 1)] # poison both calling conventions of the key + foreach(r -> ccall(:jl_insert_abi_adapter, Any, (Any,), r), recs) + @test all(r -> getfield(r, :fptr) == C_NULL, recs) + cf = @cfunction(poison_target, Any, (Cint,)) + GC.@preserve cf begin + fptr = Base.unsafe_convert(Ptr{Cvoid}, cf) + @test_throws MethodError ccall(fptr, Any, (Cint,), Cint(1)) + end + # the poisoned record for the used convention was filled in place, not shadowed + @test any(r -> getfield(r, :fptr) != C_NULL, recs) +end diff --git a/test/precompile.jl b/test/precompile.jl index 24e578d25e841..da30c6799b239 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1167,6 +1167,91 @@ precompile_test_harness("precompiletools") do dir end end +precompile_test_harness("cfunction adapter serialization") do dir + # A @cfunction whose declared C ABI differs from its target's specsig needs an ABI + # adapter. When the enclosing method is precompiled with native code, that adapter is + # compiled into the package image and its dispatch trampoline (jl_dispatch_trampoline_t) is + # serialized with the adapter wired into `fptr` -- exercising the trampoline serializer + # (jl_get_adapter_id / fptr_record) and jl_reintern_trampoline on load. Each adapter branch + # reachable from @cfunction is covered: specsig widening (well-inferred Cint boxed to `Any`) + # and narrowing (`Any`-returning target narrowed to `Cint`), const-return, the boxed args ABI + # (vararg target), and the codeinst == NULL dispatcher (both a missing method that throws and + # an abstract `Any` argument that dispatches at call time). + CFuncMod = :CFuncAdapter0x6f2c1d8a + write(joinpath(dir, "$CFuncMod.jl"), + """ + module $CFuncMod + box_target(x::Cint) = x # well-inferred Cint + any_target(x::Cint) = Core.compilerbarrier(:type, x + Cint(1)) # inferred ::Any + const_target(x::Cint) = Cint(42) # const-return + args_target(x::Cint...) = length(x) % Cint # vararg (boxed args ABI) + unresolved_target(x::Float64) = x # no method for (Cint,) + function run_widen(x::Cint) + cf = @cfunction(box_target, Any, (Cint,)) + GC.@preserve cf ccall(Base.unsafe_convert(Ptr{Cvoid}, cf), Any, (Cint,), x) + end + function run_narrow(x::Cint) + cf = @cfunction(any_target, Cint, (Cint,)) + GC.@preserve cf ccall(Base.unsafe_convert(Ptr{Cvoid}, cf), Cint, (Cint,), x) + end + function run_const(x::Cint) + cf = @cfunction(const_target, Cint, (Cint,)) + GC.@preserve cf ccall(Base.unsafe_convert(Ptr{Cvoid}, cf), Cint, (Cint,), x) + end + function run_args(x::Cint) + cf = @cfunction(args_target, Cint, (Cint,)) + GC.@preserve cf ccall(Base.unsafe_convert(Ptr{Cvoid}, cf), Cint, (Cint,), x) + end + function run_unresolved(x::Cint) + cf = @cfunction(unresolved_target, Cint, (Cint,)) + GC.@preserve cf ccall(Base.unsafe_convert(Ptr{Cvoid}, cf), Cint, (Cint,), x) + end + function run_dispatch(x::Cint) # abstract `Any` arg -> dynamic-dispatch adapter + cf = @cfunction(box_target, Any, (Any,)) + GC.@preserve cf ccall(Base.unsafe_convert(Ptr{Cvoid}, cf), Any, (Any,), x) + end + function run_dup(x::Cint) # two call sites, same (sigt, rt) -> one trampoline + cf1 = @cfunction(box_target, Any, (Cint,)) + cf2 = @cfunction(box_target, Any, (Cint,)) + r1 = GC.@preserve cf1 ccall(Base.unsafe_convert(Ptr{Cvoid}, cf1), Any, (Cint,), x) + r2 = GC.@preserve cf2 ccall(Base.unsafe_convert(Ptr{Cvoid}, cf2), Any, (Cint,), x) + (r1, r2) + end + precompile(run_widen, (Cint,)) + precompile(run_narrow, (Cint,)) + precompile(run_const, (Cint,)) + precompile(run_args, (Cint,)) + precompile(run_unresolved, (Cint,)) + precompile(run_dispatch, (Cint,)) + precompile(run_dup, (Cint,)) + end + """) + Base.compilecache(Base.PkgId(string(CFuncMod))) + M = Base.require(Main, CFuncMod) + adapter_count() = ccall(:jl_typemap_count, Csize_t, (Any,), + getfield(Core.abi_adapters, :cache)) + n0 = adapter_count() + # invokelatest: the run_* methods are defined in a world newer than this function's. + @test Base.invokelatest(M.run_widen, Cint(5)) === Cint(5) + @test Base.invokelatest(M.run_narrow, Cint(7)) == Cint(8) + @test Base.invokelatest(M.run_const, Cint(3)) == Cint(42) + @test Base.invokelatest(M.run_args, Cint(9)) == Cint(1) + @test_throws MethodError Base.invokelatest(M.run_unresolved, Cint(3)) + @test Base.invokelatest(M.run_dispatch, Cint(5)) === Cint(5) + # Two @cfunction call sites with the same (sigt, rt) share one interned trampoline, so the + # image can emit duplicate adapter records for it. Each record gets its own fvar slot (just as + # each CodeInstance does), so `fptr_record` stays 1:1 and every record's fptr is wired -- a + # shared fvar slot would instead leave one record's fptr NULL (a null call after load). + @test Base.invokelatest(M.run_dup, Cint(5)) === (Cint(5), Cint(5)) + # With pkgimage native code the adapters were compiled in and wired into the trampolines' + # `fptr`s, so these calls reuse them and never JIT a fresh adapter (the no-JIT path that + # --trim relies on). Without pkgimages nothing was precompiled to reuse, so only check the + # results above. + if Bool(Base.JLOptions().use_pkgimages) + @test adapter_count() == n0 + end +end + precompile_test_harness("invoke") do dir InvokeModule = :Invoke0x030e7e97c2365aad CallerModule = :Caller0x030e7e97c2365aad From 2404ff943bc8783ce3676702a797180895be3b94 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 15 Jul 2026 13:12:18 -0400 Subject: [PATCH 5/5] loader: add JULIA_LOAD_CODEGEN_LIB and test @cfunction without codegen Add a JULIA_LOAD_CODEGEN_LIB environment variable (0/no) that makes the loader skip libjulia-codegen and install the codegen-stubs fallbacks, exactly as when the library is absent -- so the no-codegen configuration can be tested and debugged without hiding the library file. Use it to extend the cfunction adapter serialization test with a codegen-free child process: every @cfunction call must be served by the ABI adapters serialized into the package image, and after a method redefinition the resolver must dispatch through the image-serialized dynamic-dispatch adapter rather than error. A jl_get_LLVM_VERSION probe (0 from the fallback stub) guards against the test passing vacuously through the JIT. Fixes #61949 Co-Authored-By: Claude Fable 5 --- .github/_typos.toml | 5 ++ .github/workflows/Typos.yml | 4 +- cli/loader_lib.c | 48 +++++++++++++++---- doc/src/manual/environment-variables.md | 8 ++++ test/precompile.jl | 64 ++++++++++++++++++------- 5 files changed, 102 insertions(+), 27 deletions(-) create mode 100644 .github/_typos.toml diff --git a/.github/_typos.toml b/.github/_typos.toml new file mode 100644 index 0000000000000..bcdd4d51dc563 --- /dev/null +++ b/.github/_typos.toml @@ -0,0 +1,5 @@ +# Configuration for the `typos` spell checker (.github/workflows/Typos.yml) +[default.extend-words] +# `invokee` is deliberate terminology (the resolved callee behind a dispatch +# trampoline), not a typo of "invoked"/"invoke" +invokee = "invokee" diff --git a/.github/workflows/Typos.yml b/.github/workflows/Typos.yml index 965aa3f6f5e2b..c0d3281290877 100644 --- a/.github/workflows/Typos.yml +++ b/.github/workflows/Typos.yml @@ -40,12 +40,12 @@ jobs: | tar -xz -C "${{ runner.temp }}/typos" ./typos "${{ runner.temp }}/typos/typos" --version - echo -n $NEW_FILES | xargs "${{ runner.temp }}/typos/typos" --format json >> ${{ runner.temp }}/new_typos.jsonl || true + echo -n $NEW_FILES | xargs "${{ runner.temp }}/typos/typos" --config .github/_typos.toml --format json >> ${{ runner.temp }}/new_typos.jsonl || true git checkout FETCH_HEAD -- $OLD_FILES if [ -z "$OLD_FILES" ]; then touch "${{ runner.temp }}/old_typos.jsonl" # No old files, so no old typos. else - echo -n $OLD_FILES | xargs "${{ runner.temp }}/typos/typos" --format json >> ${{ runner.temp }}/old_typos.jsonl || true + echo -n $OLD_FILES | xargs "${{ runner.temp }}/typos/typos" --config .github/_typos.toml --format json >> ${{ runner.temp }}/old_typos.jsonl || true fi diff --git a/cli/loader_lib.c b/cli/loader_lib.c index c27fbffb9993d..9f354dc12850a 100644 --- a/cli/loader_lib.c +++ b/cli/loader_lib.c @@ -117,6 +117,41 @@ static void * load_library(const char * rel_path, const char * src_dir, int err) return handle; } +// case-insensitive strcmp +static int istrcmp(const char *val, const char *token) { + for (; *token; val++, token++) { + char c = *val; + if (c >= 'A' && c <= 'Z') + c += 'a' - 'A'; + if (c != *token) + return 0; + } + return *val == '\0'; +} + +// intended to match Base.get_bool_env +static int env_var_bool(const char *name, int *value) { +#if defined(_OS_WINDOWS_) + char val[8]; + DWORD val_len = GetEnvironmentVariableA(name, val, sizeof(val)); + if (val_len == 0 || val_len >= sizeof(val)) /* unset, or too long to be a token */ + return 0; +#else + const char *val = getenv(name); + if (val == NULL) + return 0; +#endif + if (istrcmp(val, "t") || istrcmp(val, "true") || istrcmp(val, "y") || istrcmp(val, "yes") || istrcmp(val, "1")) { + *value = 1; + return 1; + } + if (istrcmp(val, "f") || istrcmp(val, "false") || istrcmp(val, "n") || istrcmp(val, "no") || istrcmp(val, "0")) { + *value = 0; + return 1; + } + return 0; +} + static void * lookup_symbol(const void * lib_handle, const char * symbol_name) { #ifdef _OS_WINDOWS_ return GetProcAddress((HMODULE) lib_handle, symbol_name); @@ -323,13 +358,7 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) { int probe_successful = 0; // Check to see if the user has disabled libstdc++ probing - char *probevar = getenv("JULIA_PROBE_LIBSTDCXX"); - if (probevar) { - if (strcmp(probevar, "1") == 0 || strcmp(probevar, "yes") == 0) - do_probe = 1; - else if (strcmp(probevar, "0") == 0 || strcmp(probevar, "no") == 0) - do_probe = 0; - } + env_var_bool("JULIA_PROBE_LIBSTDCXX", &do_probe); if (do_probe) { const char *cxxpath = libstdcxxprobe(); if (cxxpath) { @@ -362,7 +391,10 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) { libjulia_internal = load_library(curr_dep, lib_dir, 1); } else if (special_idx == 2) { // This special library is `libjulia-codegen` - libjulia_codegen = load_library(curr_dep, lib_dir, 0); + int load_codegen = 1; + env_var_bool("JULIA_LOAD_CODEGEN_LIB", &load_codegen); + if (load_codegen) + libjulia_codegen = load_library(curr_dep, lib_dir, 0); } special_idx++; } else { diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index e1f74ccb506b9..fcae0ec7d3217 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -616,3 +616,11 @@ Arguments to be passed to the LLVM backend. ### `JULIA_FALLBACK_REPL` Forces the fallback repl instead of REPL.jl. + +### `JULIA_LOAD_CODEGEN_LIB` + +If set to a false value (`0`, `f`, `false`, `n`, or `no`, case-insensitive), +the loader does not load `libjulia-codegen`, and the fallback +(interpreter-only) implementations in `libjulia-internal` are used instead — +exactly as if the library were absent from the installation. Intended for +testing and debugging the no-codegen configuration. diff --git a/test/precompile.jl b/test/precompile.jl index da30c6799b239..05a6db43b9a61 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1168,19 +1168,11 @@ precompile_test_harness("precompiletools") do dir end precompile_test_harness("cfunction adapter serialization") do dir - # A @cfunction whose declared C ABI differs from its target's specsig needs an ABI - # adapter. When the enclosing method is precompiled with native code, that adapter is - # compiled into the package image and its dispatch trampoline (jl_dispatch_trampoline_t) is - # serialized with the adapter wired into `fptr` -- exercising the trampoline serializer - # (jl_get_adapter_id / fptr_record) and jl_reintern_trampoline on load. Each adapter branch - # reachable from @cfunction is covered: specsig widening (well-inferred Cint boxed to `Any`) - # and narrowing (`Any`-returning target narrowed to `Cint`), const-return, the boxed args ABI - # (vararg target), and the codeinst == NULL dispatcher (both a missing method that throws and - # an abstract `Any` argument that dispatches at call time). CFuncMod = :CFuncAdapter0x6f2c1d8a write(joinpath(dir, "$CFuncMod.jl"), """ module $CFuncMod + llvm_version() = ccall(:jl_get_LLVM_VERSION, UInt32, ()) # 0 = no-codegen stub box_target(x::Cint) = x # well-inferred Cint any_target(x::Cint) = Core.compilerbarrier(:type, x + Cint(1)) # inferred ::Any const_target(x::Cint) = Cint(42) # const-return @@ -1224,6 +1216,7 @@ precompile_test_harness("cfunction adapter serialization") do dir precompile(run_unresolved, (Cint,)) precompile(run_dispatch, (Cint,)) precompile(run_dup, (Cint,)) + precompile(llvm_version, ()) end """) Base.compilecache(Base.PkgId(string(CFuncMod))) @@ -1238,17 +1231,54 @@ precompile_test_harness("cfunction adapter serialization") do dir @test Base.invokelatest(M.run_args, Cint(9)) == Cint(1) @test_throws MethodError Base.invokelatest(M.run_unresolved, Cint(3)) @test Base.invokelatest(M.run_dispatch, Cint(5)) === Cint(5) - # Two @cfunction call sites with the same (sigt, rt) share one interned trampoline, so the - # image can emit duplicate adapter records for it. Each record gets its own fvar slot (just as - # each CodeInstance does), so `fptr_record` stays 1:1 and every record's fptr is wired -- a - # shared fvar slot would instead leave one record's fptr NULL (a null call after load). @test Base.invokelatest(M.run_dup, Cint(5)) === (Cint(5), Cint(5)) - # With pkgimage native code the adapters were compiled in and wired into the trampolines' - # `fptr`s, so these calls reuse them and never JIT a fresh adapter (the no-JIT path that - # --trim relies on). Without pkgimages nothing was precompiled to reuse, so only check the - # results above. if Bool(Base.JLOptions().use_pkgimages) + # With pkgimage native code the adapters were compiled in and wired into the trampolines' + # `fptr`s, so these calls reuse them and never JIT a fresh adapter (the no-JIT path that + # --trim relies on). + # Without pkgimages nothing was precompiled to reuse, so only check the results above. @test adapter_count() == n0 + + # Re-run the whole adapter matrix in a build without codegen (julia#61949): + # JULIA_LOAD_CODEGEN_LIB=0 makes the loader skip libjulia-codegen and install the + # codegen-stubs fallbacks, so every call below must be served by the adapters serialized + # into the pkgimage above -- and after a method redefinition, the resolver must fall + # back to the image-serialized ci == NULL dynamic-dispatch adapter rather than error. + # The llvm=0 probe proves codegen was really absent (a top-level ccall needs the + # compiler, so the probe lives in the precompiled module). + sep = Sys.iswindows() ? ";" : ":" + script = """ + using $CFuncMod + const M = $CFuncMod + println("llvm=", M.llvm_version()) + println("widen=", M.run_widen(Cint(5))) + println("narrow=", M.run_narrow(Cint(7))) + println("const=", M.run_const(Cint(3))) + println("args=", M.run_args(Cint(9))) + println("dispatch=", M.run_dispatch(Cint(5))) + println("dup=", M.run_dup(Cint(5))) + println("unresolved=", try; M.run_unresolved(Cint(3)); "no-error"; catch e; e isa MethodError ? "MethodError" : "other-error"; end) + @eval M box_target(x::Cint) = x + Cint(100) + println("redef=", Base.invokelatest(M.run_widen, Cint(5))) + """ + outbuf = IOBuffer(); errbuf = IOBuffer() + cmd = addenv(`$(Base.julia_cmd()) --startup-file=no -e $script`, + "JULIA_LOAD_CODEGEN_LIB" => "0", + "JULIA_LOAD_PATH" => dir * sep * "@stdlib", + "JULIA_DEPOT_PATH" => first(DEPOT_PATH) * sep) + proc = run(pipeline(ignorestatus(cmd), stdout=outbuf, stderr=errbuf)) + out = String(take!(outbuf)) + success(proc) || println(stderr, "codegen-free child failed:\n", String(take!(errbuf))) + @test success(proc) + @test occursin("llvm=0", out) # codegen really was absent + @test occursin("widen=5", out) # cached specsig-widening adapter + @test occursin("narrow=8", out) # cached narrowing adapter + @test occursin("const=42", out) # cached const-return adapter + @test occursin("args=1", out) # cached boxed-args adapter + @test occursin("dispatch=5", out) # cached dynamic-dispatch adapter + @test occursin("dup=(5, 5)", out) # shared trampoline, both sites wired + @test occursin("unresolved=MethodError", out) + @test occursin("redef=105", out) # dispatcher re-query after redefinition end end