perf: inline mimalloc's allocation fast path into the runtime entry point

This PR compiles mimalloc and the fused small-object allocation entry point in a single translation unit, so that compiled code reaches memory for a small object with one call that inlines mimalloc's fast path.

The new `runtime/mimalloc.cpp` #includes all of mimalloc (`static.c`) followed by `lean_alloc_small_object_core`, replacing `static.c` as the source of the mimalloc build variants; GCC additionally needs `-fno-semantic-interposition` to inline mimalloc's exported functions under -fPIC. Second step extracted from #14950; the mimalloc source patch (`mi_malloc_small_aligned`, `MI_BLOCK_SCRUB`) and the `object_rc.cpp` split stay there.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KEqhVRwDgbXCrsn7AxsMdX

Author: TwoFX

src/include/lean/lean.h

@@ -484,9 +484,10 @@ static inline unsigned lean_get_slot_idx(unsigned sz) {
 LEAN_EXPORT void lean_inc_heartbeat(void);
 
 #ifdef LEAN_MIMALLOC
-/* Fused small-object allocation entry point implemented in `runtime/alloc.cpp`: a single call
-   covering the heartbeat update and the mimalloc allocation. Requires `sz` to be a positive
-   multiple of `LEAN_OBJECT_SIZE_DELTA` of at most `MI_SMALL_SIZE_MAX`; initializes `m_cs_sz`. */
+/* Fused small-object allocation entry point implemented in `runtime/mimalloc.cpp`: a single call
+   covering the heartbeat update and the (inlined) mimalloc fast path. Requires `sz` to be a
+   positive multiple of `LEAN_OBJECT_SIZE_DELTA` of at most `MI_SMALL_SIZE_MAX`; initializes
+   `m_cs_sz`. */
 LEAN_EXPORT LEAN_ATTR_MALLOC lean_object * lean_alloc_small_object_core(unsigned sz);
 #endif
 

src/runtime/CMakeLists.txt

@@ -1,8 +1,8 @@
 if(USE_MIMALLOC)
-  set(MIMALLOC_SRC ${LEAN_BINARY_DIR}/../mimalloc/src/mimalloc/src/static.c)
-  # (C flags are incomplete, compile as C++ instead like everything else). A source property rather
-  # than a target one, but it is the same for every variant below.
-  set_source_files_properties(${MIMALLOC_SRC} PROPERTIES LANGUAGE CXX)
+  # `mimalloc.cpp` #includes all of mimalloc (`static.c`, compiled as C++ like everything else)
+  # together with the runtime's fused allocation entry point so mimalloc's fast path inlines into
+  # it.
+  set(MIMALLOC_SRC ${CMAKE_CURRENT_SOURCE_DIR}/mimalloc.cpp)
   # Make all symbols visible, always build with optimizations as otherwise Lean becomes too slow.
   # `MI_MAX_ALIGN_SIZE=8` means that an object with two fields will take 24 bytes instead of 32 bytes.
   # The default value 16 is only necessary for mimalloc to be compatible with the system allocator, which
@@ -11,15 +11,24 @@ if(USE_MIMALLOC)
   if(CMAKE_CXX_COMPILER_ID MATCHES "AppleClang|Clang")
     list(APPEND MIMALLOC_OPTS -Wno-deprecated)
   endif()
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    # GCC refuses to inline mimalloc's exported allocation functions into the entry point when
+    # compiling with -fPIC unless told that no symbol will be interposed (clang's default)
+    list(APPEND MIMALLOC_OPTS -fno-semantic-interposition)
+  endif()
 
   # One target per (mitigation level, TLS model) the build needs, each carrying its own
-  # `target_compile_options`. The flags cannot live on the shared source instead: source properties
-  # are directory-scoped and CMake emits them *after* target options, so a target trying to override
-  # the mitigation level would silently lose to whatever the source property set.
+  # `target_compile_options`.
   function(add_mimalloc_variant name type secure)
     add_library(${name} ${type} ${MIMALLOC_SRC})
-    # Lean code includes it as `lean/mimalloc.h`, but `static.c` itself wants the original directory
-    target_include_directories(${name} PRIVATE ${LEAN_BINARY_DIR}/../mimalloc/src/mimalloc/include)
+    # Lean code includes it as `lean/mimalloc.h`, but `static.c` itself wants the original include
+    # directory, and `mimalloc.cpp` includes `static.c` from the source directory
+    target_include_directories(
+      ${name}
+      PRIVATE
+        ${LEAN_BINARY_DIR}/../mimalloc/src/mimalloc/include
+        ${LEAN_BINARY_DIR}/../mimalloc/src/mimalloc/src
+    )
     target_compile_options(${name} PRIVATE ${MIMALLOC_OPTS} -DMI_SECURE=${secure} ${ARGN})
   endfunction()
 

src/runtime/alloc.cpp

@@ -39,18 +39,4 @@ uint64_t get_num_heartbeats() {
     return lean_g_heartbeat;
 }
 
-#ifdef LEAN_MIMALLOC
-extern "C" LEAN_EXPORT LEAN_ATTR_MALLOC lean_object * lean_alloc_small_object_core(unsigned sz) {
-    lean_g_heartbeat++;
-    // the callers guarantee `sz > 0 && sz % LEAN_OBJECT_SIZE_DELTA == 0 && sz <= MI_SMALL_SIZE_MAX`
-    void * mem = mi_malloc_small(sz);
-    if (LEAN_UNLIKELY(mem == NULL)) lean_internal_panic_out_of_memory();
-    lean_object * o = (lean_object *)mem;
-    /* `m_cs_sz` must be the exact (aligned) requested size, not mimalloc's potentially larger
-       block size: `lean_small_object_size` and `leangz` rely on it. */
-    o->m_cs_sz = sz;
-    return o;
-}
-#endif
-
 }

src/runtime/mimalloc.cpp

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2026 Lean FRO, LLC. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+
+Author: Julia M. Himmel
+*/
+/*
+Compiles all of mimalloc (`static.c`) together with the runtime's small-object allocation entry
+point in a single translation unit so that mimalloc's allocation fast path inlines into it:
+compiled Lean code reaches the allocator, including the heartbeat update, with a single call.
+`static.c` must come first so that mimalloc's headers configure themselves from this variant's
+`MI_*` flags (see `add_mimalloc_variant` in `CMakeLists.txt`) before `lean/mimalloc.h` is seen.
+*/
+#include <static.c>
+// mimalloc's `atomic.h` already defined `_Atomic` for C++; `lean.h` redefines it identically
+#undef _Atomic
+#include <lean/lean.h>
+#include "runtime/alloc.h"
+
+extern "C" LEAN_EXPORT LEAN_ATTR_MALLOC lean_object * lean_alloc_small_object_core(unsigned sz) {
+    lean_g_heartbeat++;
+    // the callers guarantee `sz > 0 && sz % LEAN_OBJECT_SIZE_DELTA == 0 && sz <= MI_SMALL_SIZE_MAX`
+    void * mem = mi_malloc_small(sz);
+    if (LEAN_UNLIKELY(mem == NULL)) lean_internal_panic_out_of_memory();
+    lean_object * o = (lean_object *)mem;
+    /* `m_cs_sz` must be the exact (aligned) requested size, not mimalloc's potentially larger
+       block size: `lean_small_object_size` and `leangz` rely on it. */
+    o->m_cs_sz = sz;
+    return o;
+}