diff --git a/doc/reflections/das2rst.das b/doc/reflections/das2rst.das index 347971e3b4..b953494c35 100644 --- a/doc/reflections/das2rst.das +++ b/doc/reflections/das2rst.das @@ -216,6 +216,7 @@ def document_module_fio(_root : string) { var mod = [get_module("fio_core"), find_module("fio")] var groups <- array( hide_group(group_by_regex("Internal builtin functions", mod, %regex~builtin%%)), + group_by_regex("Direct IO and mapping advice", mod, %regex~(dwrite_open|dwrite_append|dwrite_band|dwrite_commit|dwrite_stat|dwrite_close|prefetch_map)$%%), group_by_regex("File manipulation", mod, %regex~(f|long_f|stat$|getchar|remove|rename|copy_file|file_size|equivalent|is_symlink|set_mtime)%%), group_by_regex("Path manipulation", mod, %regex~(base_name|dir_name|get_full_file_name|extension|stem|replace_extension|path_join|normalize|to_generic_path|is_absolute|relative|relative_result|parent)$%%), group_by_regex("Directory manipulation", mod, %regex~(dir|dir_rec|mkdir|mkdir_rec|mkdir_result|rmdir|rmdir_rec|rmdir_rec_result|rmdir_result|chdir|getcwd)$%%), diff --git a/doc/source/stdlib/handmade/function-fio-dwrite_append-0x6658e660a37b258d.rst b/doc/source/stdlib/handmade/function-fio-dwrite_append-0x6658e660a37b258d.rst new file mode 100644 index 0000000000..e5048ab537 --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-dwrite_append-0x6658e660a37b258d.rst @@ -0,0 +1 @@ +Append ``bytes`` from ``data`` at the writer's current end. Any pointer, any size — the handle re-blocks into aligned bands internally, and a band-multiple, band-aligned source is written directly with no copy. Returns false when a write fails (disk full, in practice); a failed writer stays failed, so checking ``dwrite_close`` once is enough. diff --git a/doc/source/stdlib/handmade/function-fio-dwrite_band-0x160228ade96e97be.rst b/doc/source/stdlib/handmade/function-fio-dwrite_band-0x160228ade96e97be.rst new file mode 100644 index 0000000000..0873f25091 --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-dwrite_band-0x160228ade96e97be.rst @@ -0,0 +1 @@ +Hand out the writer's own staging band so a producer can build bytes in place instead of building a buffer and appending a copy. ``avail`` receives the free space left in the current band; fill up to that many bytes and account for them with ``dwrite_commit``. Returns null on a dead writer. diff --git a/doc/source/stdlib/handmade/function-fio-dwrite_close-0xca8d8214309caf7.rst b/doc/source/stdlib/handmade/function-fio-dwrite_close-0xca8d8214309caf7.rst new file mode 100644 index 0000000000..ee9291a4e0 --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-dwrite_close-0xca8d8214309caf7.rst @@ -0,0 +1 @@ +Flush the writer's tail (sector-padded on cache-bypassing platforms), truncate the file to exactly what was appended, and free the handle. Returns false if anything failed anywhere along the writer's life — this is the one completion verdict a caller needs to check. diff --git a/doc/source/stdlib/handmade/function-fio-dwrite_commit-0x122d2094a7b99d77.rst b/doc/source/stdlib/handmade/function-fio-dwrite_commit-0x122d2094a7b99d77.rst new file mode 100644 index 0000000000..8be300795a --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-dwrite_commit-0x122d2094a7b99d77.rst @@ -0,0 +1 @@ +Account ``bytes`` written into the pointer ``dwrite_band`` handed out; a full band flushes to the device automatically. Returns false when the commit overruns the band's remaining space (a producer bug) or the writer already failed. diff --git a/doc/source/stdlib/handmade/function-fio-dwrite_open-0x46f123a0dc4b0ce0.rst b/doc/source/stdlib/handmade/function-fio-dwrite_open-0x46f123a0dc4b0ce0.rst new file mode 100644 index 0000000000..7a4de5fa46 --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-dwrite_open-0x46f123a0dc4b0ce0.rst @@ -0,0 +1 @@ +Open ``path`` for cache-bypassing sequential append (``FILE_FLAG_NO_BUFFERING`` on Windows, ``F_NOCACHE`` on macOS, ``posix_fallocate`` plus ``fadvise`` elsewhere). ``total_bytes`` preallocates the file — writes must then be strictly ascending, which is what avoids NTFS valid-data-length zero-fill; the file is truncated back to what was actually appended at close. ``band_bytes`` sizes the internal sector-aligned bounce buffer (16 MB when in doubt). Returns null on failure. Pair with ``dwrite_append`` or the ``dwrite_band``/``dwrite_commit`` producer surface, and always finish with ``dwrite_close``. diff --git a/doc/source/stdlib/handmade/function-fio-dwrite_stat-0xbf270ab2cb2df9d6.rst b/doc/source/stdlib/handmade/function-fio-dwrite_stat-0xbf270ab2cb2df9d6.rst new file mode 100644 index 0000000000..a6b75d743d --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-dwrite_stat-0xbf270ab2cb2df9d6.rst @@ -0,0 +1 @@ +Where the writer spent its time so far: ``which`` 0 = staging nanoseconds (memcpy into the bounce band), 1 = syscall nanoseconds, 2 = bytes written directly from caller memory, 3 = bytes that went through the bounce band. Call before ``dwrite_close`` — the handle is gone after. diff --git a/doc/source/stdlib/handmade/function-fio-fmap_open_rw-0x2abb9e9d0dfc252e.rst b/doc/source/stdlib/handmade/function-fio-fmap_open_rw-0x2abb9e9d0dfc252e.rst new file mode 100644 index 0000000000..63a30009cc --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-fmap_open_rw-0x2abb9e9d0dfc252e.rst @@ -0,0 +1 @@ +The writable twin of ``fmap_open``: memory-maps the file at ``path`` as a SHARED read-write view and returns the mapping's base pointer, writing the mapped byte count through ``size``. Writes through the mapping go back to the file itself via the page cache — this is a real shared mapping, not copy-on-write, so no commit charge is taken for the view. The mapping stays valid until passed to ``fmap_close``. Returns null when the file cannot be opened for writing, is empty, or cannot be mapped. diff --git a/doc/source/stdlib/handmade/function-fio-prefetch_map-0xc63f5d4e91b2e653.rst b/doc/source/stdlib/handmade/function-fio-prefetch_map-0xc63f5d4e91b2e653.rst new file mode 100644 index 0000000000..8ade89dd63 --- /dev/null +++ b/doc/source/stdlib/handmade/function-fio-prefetch_map-0xc63f5d4e91b2e653.rst @@ -0,0 +1 @@ +Advisory readahead over a mapped range: asks the OS to fault ``bytes`` starting at ``base`` into the page cache ahead of use (``PrefetchVirtualMemory`` on Windows, ``madvise(MADV_WILLNEED)`` on POSIX). Use it on an ``fmap``/``fmap_open`` mapping before a parallel pass over the data — on-demand faults taken inside worker lanes serialize them. Returns false when the OS declines; the call is purely advisory, so reads work either way. diff --git a/include/daScript/simulate/aot_builtin_fio.h b/include/daScript/simulate/aot_builtin_fio.h index 6a31b084a3..d8d2e982f5 100644 --- a/include/daScript/simulate/aot_builtin_fio.h +++ b/include/daScript/simulate/aot_builtin_fio.h @@ -88,7 +88,15 @@ namespace das { DAS_API vec4f builtin_load ( Context & context, SimNode_CallBase *, vec4f * args ); DAS_API void builtin_map_file ( const FILE* _f, const TBlock>>& blk, Context*, LineInfoArg * at ); DAS_API void * builtin_fmap_open ( const char * name, uint64_t * size, Context * context, LineInfoArg * at ); + DAS_API void * builtin_fmap_open_rw ( const char * name, uint64_t * size, Context * context, LineInfoArg * at ); DAS_API void builtin_fmap_close ( void * data, uint64_t size, Context * context, LineInfoArg * at ); + DAS_API void * builtin_dwrite_open ( const char * name, uint64_t total_bytes, uint64_t band_bytes, Context * context, LineInfoArg * at ); + DAS_API bool builtin_dwrite_append ( void * h, void * data, uint64_t bytes, Context * context, LineInfoArg * at ); + DAS_API void * builtin_dwrite_band ( void * h, uint64_t * avail, Context * context, LineInfoArg * at ); + DAS_API bool builtin_dwrite_commit ( void * h, uint64_t bytes, Context * context, LineInfoArg * at ); + DAS_API uint64_t builtin_dwrite_stat ( void * h, int32_t which, Context * context, LineInfoArg * at ); + DAS_API bool builtin_dwrite_close ( void * h, Context * context, LineInfoArg * at ); + DAS_API bool builtin_prefetch_map ( void * base, uint64_t bytes, Context * context, LineInfoArg * at ); DAS_API char * builtin_dirname ( const char * name, Context * context, LineInfoArg * at ); DAS_API char * builtin_basename ( const char * name, Context * context, LineInfoArg * at ); DAS_API bool builtin_fstat ( const FILE * f, FStat & fs, Context * context, LineInfoArg * at ); diff --git a/modules/dasLLAMA/.das_module b/modules/dasLLAMA/.das_module index 077ed5758d..dc88dd079a 100644 --- a/modules/dasLLAMA/.das_module +++ b/modules/dasLLAMA/.das_module @@ -4,7 +4,7 @@ require daslib/fio [export] def initialize(project_path : string) { let dasllama_paths = [ - "dasllama_env", "dasllama_par", "dasllama_parity", "dasllama_tune", "dasllama_math", "dasllama_math_default", "dasllama_math_aarch64_neon", "dasllama_math_gen", "dasllama_math_metal", "dasllama_math_vulkan", "dasllama_math_accelerate", "dasllama_kernel_access", "dasllama_vulkan_lens", "dasllama_metal_lens", "dasllama_metal_common", "dasllama_metal_kernels", "dasllama_metal_prefill", "dasllama_metal_llama", "dasllama_gemm_gen", "dasllama_gemm_register", "dasllama_gemm_schema", "dasllama_quant", "dasllama_gguf", + "dasllama_env", "dasllama_config", "dasllama_par", "dasllama_parity", "dasllama_tune", "dasllama_math", "dasllama_math_default", "dasllama_math_aarch64_neon", "dasllama_math_gen", "dasllama_math_metal", "dasllama_math_vulkan", "dasllama_math_accelerate", "dasllama_kernel_access", "dasllama_vulkan_lens", "dasllama_metal_lens", "dasllama_metal_shapes", "dasllama_metal_common", "dasllama_metal_kernels", "dasllama_metal_prefill", "dasllama_metal_llama", "dasllama_gemm_gen", "dasllama_gemm_register", "dasllama_gemm_schema", "dasllama_quant", "dasllama_gguf", "dasllama_unicode", "dasllama_tokenizer", "dasllama_bpe", "dasllama_common", "dasllama_layout", "dasllama_prefix", "dasllama_audio", "dasllama_audio_io", "dasllama_whisper", "dasllama_qwen3a", "dasllama_parakeet", "dasllama_canary", "dasllama_gemma4a", "dasllama_vad", "dasllama_asr", "dasllama_arch_llama", "dasllama_arch_qwen2", "dasllama_arch_qwen3", "dasllama_arch_phi3", "dasllama_arch_gemma2", "dasllama_arch_gemma3", "dasllama_arch_gemma4", "dasllama_arch_qwen2moe", "dasllama_arch_qwen3moe", "dasllama_arch_qwen35", "dasllama_arch_gptoss", "dasllama_arch_mistral3", "dasllama_arch_glm4moe", "dasllama_image", "dasllama_transformer", "dasllama_chat", "dasllama" diff --git a/modules/dasLLAMA/ENVIRONMENT.md b/modules/dasLLAMA/ENVIRONMENT.md index 3baefe49dd..dbdd8122e4 100644 --- a/modules/dasLLAMA/ENVIRONMENT.md +++ b/modules/dasLLAMA/ENVIRONMENT.md @@ -46,6 +46,7 @@ Read by the inference engine itself, so these affect any program that loads a mo | `DASLLAMA_GPU_COMBINE` | flag | on | Device-side routed MoE combine; 0 falls back to the host combine. | | `DASLLAMA_GPU_HEAT` | number | 0 | Expert heat threshold: hold the N hottest experts resident regardless of layer placement. | | `DASLLAMA_GPU_PROF` | flag | off | Report lifetime GPU queue submissions (real commands plus staging round-trips). | +| `DASLLAMA_PREFETCH` | flag | on | Advisory source-mapping readahead at gguf load (cold-conversion fix); =0 restores on-demand faulting. | ## Metal backend @@ -95,6 +96,8 @@ Vulkan GPU backend. Present only where the dasVulkan package is installed. | `DASLLAMA_MM_SMALLD` | number | 64 | Small-d cutoff routing narrow roles (k/v) to the small tier; widening measured worse, so this is an instrument. | | `DASLLAMA_VK_FUSE` | flag | on | Fused decode tail (add+rms+requant, qk-norm+rope); 0 pins the split dispatches for a same-build A/B. | | `DASLLAMA_VK_XFERQ` | flag | on | Stream expert uploads on the dedicated transfer queue, overlapped via a timeline semaphore; 0 keeps the single-queue rail. | +| `DASLLAMA_VK_IMPORT` | flag | on | Stream mirrors import the mapped .dlim (VK_EXT_external_memory_host) instead of pinned copies; =0 restores the copy path. | +| `DASLLAMA_TRIM` | flag | off | Serve from P3-trimmed vulkan images (big CPU weight families dropped; folded into the flavor identity). | | `DASLLAMA_VK_MEMPRIO` | flag | on | Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours. | | `DASLLAMA_VK_FA` | flag | on | Vulkan flash-attention kernel; 0 falls back to the chunked path. | | `DASLLAMA_VK_REBAR` | flag | on | Use a ReBAR device-local host-visible heap when one larger than 1GB is present. | @@ -187,6 +190,7 @@ Apple Accelerate / AMX float lane. `DASLLAMA_ACCEL` arms the whole group. | `DASLLAMA_WHISPER_DIR` | path | unset | Directory of whisper models for the audio tests. | | `DASLLAMA_CORPUS_DIR` | path | unset | Directory of audio corpus files for the transcription tests. | | `TMPDIR` | path | /tmp | Scratch directory for test artifacts; set by the OS on macOS. | +| `TEMP` | path | unset | Windows scratch-directory fallback when TMPDIR is unset (the test runner's log dir). | ## Examples and tutorials diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index f7922ef6e3..7fc1b02afd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -5,6 +5,7 @@ module dasllama_common shared public require daslib/fio require dasllama/dasllama_env // the shared typed env readers + the knob registry +require dasllama/dasllama_config // DlimConfiguration — this module registers the cpu/metal sources require daslib/array_boost // nolint:STYLE030 — temp_array is [template]; STYLE030 misses template-fn refs and false-flags this as unused require dasllama/dasllama_math require dasllama/dasllama_math_default // registers the portable Q8·Q8 backend at [init] (the fallback everywhere) @@ -388,8 +389,11 @@ def private repack_q51_stacks(var t : Model) { // flavor load's walk skips the transforms and copies the mapped blob back, plan-checked. //! One baked gather of the vulkan flavor's plan — walk order, offsets into Model.vkblob. +//! `role` records WHICH walk stage produced it (VkBakeRole) — the trim pass (P3) and the +//! slice verifier both key on it. struct VkPlanEntry { fmt : int + role : int woff : int64 n : int64 rows : int64 @@ -399,6 +403,20 @@ struct VkPlanEntry { ws_bytes : int64 } +//! The upload-walk stage a baked gather belongs to — set by the walk, stamped per entry. +enum VkBakeRole { + none + cls + dense + shexp + dn + attn + qkv + expert //!< resident expert stack (VRAM) + expert_stream //!< streamed expert stack (pinned mirror; CPU planes stay — decode runs there) + arena //!< resident-driver arena plane (dense models) +} + enum private VkBakeMode { off collect @@ -409,10 +427,39 @@ var private g_vk_bake = VkBakeMode.off var private g_vk_bake_blob : array var private g_vk_bake_plan : array var private g_vk_bake_cursor = 0l +var private g_vk_bake_role = VkBakeRole.none + +// P2 imported mirrors: the mapped-blob window of each sliced gather, keyed woff*8+fmt (the +// marks convention) — the stream-mirror path imports these instead of copying to pinned RAM +struct private SliceWindow { + wq : void? + ws : void? + wq_bytes : int64 + ws_bytes : int64 +} + +var private g_vk_slice_windows : table + +def private vk_slice_window_provider(woff : int64; fmt : int) : MoeGpuSliceWindow { + let k = woff * 8l + int64(fmt) + if (!key_exists(g_vk_slice_windows, k)) { + return default + } + unsafe { + let w & = g_vk_slice_windows[k] + return (wq = w.wq, ws = w.ws, wqb = w.wq_bytes, wsb = w.ws_bytes) + } +} + +//! The upload walk declares its current stage here (collect stamps it, slice verifies it). +def vulkan_bake_role(r : VkBakeRole) { + g_vk_bake_role = r +} //! Arm collection for the next model load's GPU walk (the vulkan-flavor bake). def vulkan_bake_begin { g_vk_bake = VkBakeMode.collect + g_vk_bake_role = VkBakeRole.none g_vk_bake_blob |> clear() g_vk_bake_plan |> clear() } @@ -432,7 +479,9 @@ def vulkan_bake_take(var t : Model) : bool { //! Arm slicing for a mapped vulkan-flavor load: the walk reads t.vkblob instead of gathering. def vulkan_bake_slice_begin { g_vk_bake = VkBakeMode.slice + g_vk_bake_role = VkBakeRole.none g_vk_bake_cursor = 0l + g_vk_slice_windows |> clear() } //! Disarm after a successful flavor load; a partially-consumed plan means the walk diverged @@ -469,11 +518,13 @@ def private vk_bake_append(a : array) : int64 { return off } +// Called AFTER the tier accepted the upload/placement (a refused gather must not enter the +// plan — the slice path would replay the refusal's absence and diverge). def private vk_bake_collect(fmt : int; woff, n, rows : int64; wq, ws : array) { if (g_vk_bake != VkBakeMode.collect) { return } - var e = VkPlanEntry(fmt = fmt, woff = woff, n = n, rows = rows, + var e = VkPlanEntry(fmt = fmt, role = int(g_vk_bake_role), woff = woff, n = n, rows = rows, wq_bytes = long_length(wq), ws_bytes = long_length(ws)) e.wq_off = vk_bake_append(wq) e.ws_off = vk_bake_append(ws) @@ -485,19 +536,36 @@ def private vk_bake_slice(t : Model; fmt : int; woff, n, rows : int64; if (g_vk_bake != VkBakeMode.slice) { return false } - // plan divergence DECLINES to live gathers mid-walk (seamless: matched entries were byte-identical) + // the bake records only ACCEPTED gathers — serve a would-refuse mismatch EMPTY (the refusal is pure arithmetic), hold the cursor: the walk re-refuses, rolls back, and re-syncs with the plan if (g_vk_bake_cursor >= long_length(t.vkplan)) { + if (!moe_gpu_would_accept(n, rows, fmt, g_vk_bake_role == VkBakeRole.expert_stream)) { + wq |> resize(0) + ws |> resize(0) + return true + } to_log(LOG_WARNING, "dasLLAMA vulkan image: the GPU walk overran the baked plan at gather {g_vk_bake_cursor} — regathering live from here\n") g_vk_bake = VkBakeMode.off return false } let e = t.vkplan[int(g_vk_bake_cursor)] - if (e.fmt != fmt || e.woff != woff || e.n != n || e.rows != rows) { - to_log(LOG_WARNING, "dasLLAMA vulkan image: baked gather {g_vk_bake_cursor} is (fmt {e.fmt} woff {e.woff} {e.n}x{e.rows}), the walk wants (fmt {fmt} woff {woff} {n}x{rows}) — regathering live from here\n") + if (e.fmt != fmt || e.woff != woff || e.n != n || e.rows != rows || e.role != int(g_vk_bake_role)) { + if (!moe_gpu_would_accept(n, rows, fmt, g_vk_bake_role == VkBakeRole.expert_stream)) { + wq |> resize(0) + ws |> resize(0) + return true + } + to_log(LOG_WARNING, "dasLLAMA vulkan image: baked gather {g_vk_bake_cursor} is (fmt {e.fmt} role {e.role} woff {e.woff} {e.n}x{e.rows}), the walk wants (fmt {fmt} role {int(g_vk_bake_role)} woff {woff} {n}x{rows}) — regathering live from here\n") g_vk_bake = VkBakeMode.off return false } g_vk_bake_cursor++ + // record the mapped window (P2: the stream-mirror path imports it instead of copying) + unsafe { + g_vk_slice_windows |> insert(e.woff * 8l + int64(fmt), SliceWindow( + wq = e.wq_bytes > 0l ? addr(t.vkblob[e.wq_off]) : null, + ws = e.ws_bytes > 0l ? addr(t.vkblob[e.ws_off]) : null, + wq_bytes = e.wq_bytes, ws_bytes = e.ws_bytes)) + } // int-indexed destinations: a >2GB plane would resize truncated and then memcpy past it assert(e.wq_bytes <= 2_147_483_647l && e.ws_bytes <= 2_147_483_647l, "vulkan image: a baked plane exceeds the int-indexed array cap") @@ -521,8 +589,10 @@ def private moe_gpu_gather_stack(t : Model; woff : int64; n, rows, slice_rows : if (vk_bake_slice(t, 0, woff, n, rows, wq, ws)) { return } + if (t.planes_trimmed) { + panic("dasLLAMA: planes_trimmed image — live gather impossible (baked plan diverged, or a re-gather was requested); rebake without --trim or match the image's config") + } invoke(g_layout_gather, t, woff, n, rows, slice_rows, mr, kgroup, wbias, wq, ws) - vk_bake_collect(0, woff, n, rows, wq, ws) } def private moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; @@ -531,8 +601,10 @@ def private moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, row if (vk_bake_slice(t, int(fmt), woff, n, rows, wq, ws)) { return } + if (t.planes_trimmed) { + panic("dasLLAMA: planes_trimmed image — live gather impossible (baked plan diverged, or a re-gather was requested); rebake without --trim or match the image's config") + } invoke(g_layout_gather_kq, t, fmt, woff, n, rows, slice_rows, repacked, mr, wq, ws) - vk_bake_collect(int(fmt), woff, n, rows, wq, ws) } def private moe_gpu_gather_stack_q8n(t : Model; woff : int64; n, rows, slice_rows : int64; @@ -542,8 +614,93 @@ def private moe_gpu_gather_stack_q8n(t : Model; woff : int64; n, rows, slice_row if (vk_bake_slice(t, 6, woff, n, rows, wq, no_ws)) { return } + if (t.planes_trimmed) { + panic("dasLLAMA: planes_trimmed image — live gather impossible (baked plan diverged, or a re-gather was requested); rebake without --trim or match the image's config") + } invoke(g_layout_gather_q8n, t, woff, n, rows, slice_rows, mr, kgroup, wbias, wq) - vk_bake_collect(6, woff, n, rows, wq, no_ws) +} + +//! P3 trim (generate-then-free): pack the emb/cls region into embq/embs, mark the model +//! trimmed, and FREE the big CPU weight families — the save then writes them as empty planes. +//! Arena-only plans qualify (pure resident-driver dense); false = declined, nothing changed. +def trim_model_planes(var t : Model) : bool { + if (empty(t.vkplan) || t.config.n_layer_nextn > 0l) { + return false + } + for (e in t.vkplan) { + if (e.role != int(VkBakeRole.arena)) { + to_log(LOG_WARNING, "dasLLAMA trim: baked plan carries non-arena role {e.role} — only pure resident-driver dense models trim; declining\n") + return false + } + } + let dim = t.config.dim + let vocab = t.config.vocab_size + if (t.cls_q8) { + // mblob-style 34B interleaved blocks over the emb region (dequant_embq_row's layout) + let nb = vocab * dim / 32l + t.embq |> resize(int(nb * 34l)) + unsafe { + for (b in range64(nb)) { + let off = t.emb_q8_off + b * 32l + var bp = addr(t.embq[b * 34l]) + let sc = t.wscale_f16 ? uint(t.qscales16[off / 32l]) : f32_to_f16(t.qscales[off / 32l]) + bp[0] = uint8(sc & 0xFFu) + bp[1] = uint8((sc >> 8u) & 0xFFu) + memcpy(addr(t.embq[b * 34l + 2l]), addr(t.qblob[off]), 32l) + } + } + } elif (cls_kq(t)) { + // the region's plane pair verbatim (repack layout preserved), rebased to superblock 0 + let nsb = vocab * (dim / 256l) + let sb0 = t.wcls_off / 256l + unsafe { + if (t.emb_fmt == KqFmt.k4) { + t.embq |> resize(int(nsb * K4_QSB)) + t.embs |> resize(int(nsb * K4_SSB)) + memcpy(addr(t.embq[0]), addr(t.k4q[sb0 * K4_QSB]), nsb * K4_QSB) + memcpy(addr(t.embs[0]), addr(t.k4s[sb0 * K4_SSB]), nsb * K4_SSB) + } elif (t.emb_fmt == KqFmt.k5) { + t.embq |> resize(int(nsb * K5_QSB)) + t.embs |> resize(int(nsb * K5_SSB)) + memcpy(addr(t.embq[0]), addr(t.k5q[sb0 * K5_QSB]), nsb * K5_QSB) + memcpy(addr(t.embs[0]), addr(t.k5s[sb0 * K5_SSB]), nsb * K5_SSB) + } elif (t.emb_fmt == KqFmt.k6) { + t.embq |> resize(int(nsb * K6_QSB)) + t.embs |> resize(int(nsb * K6_SSB)) + memcpy(addr(t.embq[0]), addr(t.k6q[sb0 * K6_QSB]), nsb * K6_QSB) + memcpy(addr(t.embs[0]), addr(t.k6s[sb0 * K6_SSB]), nsb * K6_SSB) + } elif (t.emb_fmt == KqFmt.q40) { + t.embq |> resize(int(nsb * Q40_QSB)) + t.embs |> resize(int(nsb * Q40_SSB)) + memcpy(addr(t.embq[0]), addr(t.q40q[sb0 * Q40_QSB]), nsb * Q40_QSB) + memcpy(addr(t.embs[0]), addr(t.q40s[sb0 * Q40_SSB]), nsb * Q40_SSB) + } else { + to_log(LOG_WARNING, "dasLLAMA trim: emb fmt '{t.emb_fmt}' has no kq plane pair — declining\n") + return false + } + } + } // fp32 embeddings ride fblob, which the trim keeps + t.planes_trimmed = true + delete t.qblob + delete t.qscales + delete t.qscales16 + delete t.wblob + delete t.bf16blob + delete t.k4q + delete t.k4s + delete t.k5q + delete t.k5s + delete t.k6q + delete t.k6s + delete t.q40q + delete t.q40s + delete t.q51q + delete t.q51s + delete t.mxq + delete t.mxs + delete t.q4blob + delete t.q4scales + return true } def register_batch_decode_override(name : string; fn : BatchDecodeOverrideFn) { @@ -617,6 +774,10 @@ def private blob_only_panic(t : Model; what : string) { if (t.metal_blob) { panic("dasLLAMA: CPU {what} on a blob-only metal-flavor model — serve it on the GPU (set_metal_mode / fix the declined gate) or load the CPU-flavor image (metal mode off at load_model)") } + if (t.planes_trimmed) { + // the resident overrides need an f32-KV session — the commonest decline on a trimmed image + panic("dasLLAMA: CPU {what} on a planes-trimmed vulkan-flavor image — the resident driver must serve it (create the session with KVDtype.f32, keep the tier armed) or rebake without --trim") + } } def has_decode_override(name : string) : bool { @@ -772,6 +933,9 @@ struct Model { // scale strips, k6s the GPU split form. CPU inference on a blob-only model PANICS. mblob : array metal_blob : bool = false + // P3-trimmed image: the big CPU weight families are EMPTY (GPU-resident consumers only); + // embed reads embq/embs, re-gathers must slice from vkblob, CPU fallbacks fail closed + planes_trimmed : bool = false q4blob : array q4scales : array // Native-MXFP4 expert stacks (valid when experts_mx4): the routed we1/we2/we3 stacks live here @@ -927,6 +1091,11 @@ struct Model { // views on a flavor load — the generic finalizer walk covers both. vkblob : array vkplan : array + // trimmed-flavor embedding planes (P3): the emb/cls tensor's rows kept CPU-readable after + // the big weight families drop. q8: mblob-style 34B interleaved blocks (embs empty); + // kq: the region's quant + scale planes copied verbatim (repack layout preserved, rebased to 0). + embq : array + embs : array // prepared-model image backing (dasllama_image): when set, every locked array field above // is a borrowed VIEW into this mapping (zero-copy) and the finalizer below unmaps it. // Both fields are skipped by the image save/load field walk. @@ -952,6 +1121,8 @@ def finalize(var t : Model) { } } if (t.image_map != null) { + // imported device memory over this mapping (P2 mirrors) must release BEFORE the unmap + moe_gpu_unmap_notify(t.image_map, t.image_bytes) unsafe { fmap_close(t.image_map, t.image_bytes) } @@ -970,7 +1141,10 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.mxq) + long_length(t.mxs) + long_length(t.bf16blob) * 2l + long_length(t.k4q) + long_length(t.k4s) + long_length(t.k5q) + long_length(t.k5s) - + long_length(t.k6q) + long_length(t.k6s) + long_length(t.q40q) + long_length(t.q40s)) + + long_length(t.k6q) + long_length(t.k6s) + long_length(t.q40q) + long_length(t.q40s) + + long_length(t.q51q) + long_length(t.q51s) + + long_length(t.mblob) + long_length(t.vkblob) + + long_length(t.embq) + long_length(t.embs)) } // ===== Arch registry: per-architecture blocks + chat template ===== @@ -1538,7 +1712,322 @@ def conv_profile_report() { delete rows } +// ===== streaming plan/fill (phase D of the .dlim arc) ===== + +//! One recorded big-tensor conversion: load_big's call args, replayable — the fill pass +//! re-derives the branch from (fmt, quant, disk type), so plan and fill cannot drift. +struct FillJob { + name : string + woff : int64 + n : int64 + src_off : int64 + fmt : KqFmt +} + +//! One repack region over a plane (element offset + 2D shape). fmt ids follow the repack +//! walkers: 0 = q8, 1 = mx4, 2 = q51, 4/5/6/40 = the kq plane pairs. +struct StreamReg { + off : int64 + n : int64 + d : int64 + fmt : int +} + +typedef StreamCollectFn = function<(t : Model; var regs : array) : void> +typedef StreamRepackFn = function<(fmt : int; qp, sp : void?; n, d : int64) : void> + +[unused_argument(t, regs)] +def private stream_no_collect(t : Model; var regs : array) { + panic("dasLLAMA: no stream-layout hooks registered (dasllama_layout's [init] provides them)") +} + +[unused_argument(fmt, qp, sp, n, d)] +def private stream_no_repack(fmt : int; qp, sp : void?; n, d : int64) { + panic("dasLLAMA: no stream-repack hook registered (dasllama_layout's [init] provides it)") +} + +var private g_stream_collect_q8 = @@stream_no_collect +var private g_stream_collect_kq = @@stream_no_collect +var private g_stream_collect_q51 = @@stream_no_collect +var private g_stream_repack = @@stream_no_repack + +//! dasllama_layout registers the streaming-fill collaborators at [init]: the three region +//! collectors (the repack walkers' build halves) and the single-region repack dispatcher. +def register_stream_layout(cq8, ckq, cq51 : StreamCollectFn; rr : StreamRepackFn) { + g_stream_collect_q8 = cq8 + g_stream_collect_kq = ckq + g_stream_collect_q51 = cq51 + g_stream_repack = rr +} + +var private g_fill_plan_mode = false +var private g_fill_plan : array +// field -> the plane's LAYOUT byte total, recorded at sizing: a layout can hold allocated-but- +// never-written holes (glm4moe's dense-lead shexp slices), so the streamed plane must be sized +// and gap-filled from the layout — a job-sum concatenation SHIFTS everything past the first hole +var private g_stream_plane_total : table + +//! The regions the eager repack stage WOULD build for `t` — same gates, same walkers (the +//! collector halves), so the streamed fill's per-job repack cannot drift from the eager one. +def stream_collect_regions(t : Model; var regs : array) { + if (t.quant != QuantMode.q8 || !select_matmul_backend_for_load()) { + return + } + invoke(g_stream_collect_q8, t, regs) + if (active_q51_layout_mr() > 1l) { + invoke(g_stream_collect_q51, t, regs) + } + if (t.kquant_native && kernel_backend_has_kq()) { + invoke(g_stream_collect_kq, t, regs) + } +} + +// the streamed destination field of a job (Model plane field name), "" = not streamed +def private stream_field_of(t : Model; fmt : KqFmt) : string { + if (fmt == KqFmt.k4) { + return "k4q" + } elif (fmt == KqFmt.k5) { + return "k5q" + } elif (fmt == KqFmt.k6) { + return "k6q" + } elif (fmt == KqFmt.q40) { + return "q40q" + } elif (fmt == KqFmt.q51) { + return "q51q" + } elif (t.quant == QuantMode.q8) { + return "qblob" + } + return "" +} + +// (q plane bytes, scale plane bytes, region fmt id) for one streamed job +[unused_argument(t)] +def private stream_job_shape(t : Model; j : FillJob) : tuple { + if (j.fmt == KqFmt.k4) { + return (qb = (j.n / 256l) * K4_QSB, sb = (j.n / 256l) * K4_SSB, rfmt = 4) + } elif (j.fmt == KqFmt.k5) { + return (qb = (j.n / 256l) * K5_QSB, sb = (j.n / 256l) * K5_SSB, rfmt = 5) + } elif (j.fmt == KqFmt.k6) { + return (qb = (j.n / 256l) * K6_QSB, sb = (j.n / 256l) * K6_SSB, rfmt = 6) + } elif (j.fmt == KqFmt.q40) { + return (qb = (j.n / 256l) * Q40_QSB, sb = (j.n / 256l) * Q40_SSB, rfmt = 40) + } elif (j.fmt == KqFmt.q51) { + return (qb = (j.n / 32l) * Q51_QB, sb = (j.n / 32l) * Q51_SB, rfmt = 2) + } + return (qb = j.n, sb = 0l, rfmt = 0) // q8: 1 byte/elem quants; scales handled as floats +} + +// a job's byte offset within its plane (woff is elems; kq/q51 planes stride per superblock) +def private stream_job_qoff(j : FillJob) : int64 { + if (j.fmt == KqFmt.k4) { + return (j.woff / 256l) * K4_QSB + } elif (j.fmt == KqFmt.k5) { + return (j.woff / 256l) * K5_QSB + } elif (j.fmt == KqFmt.k6) { + return (j.woff / 256l) * K6_QSB + } elif (j.fmt == KqFmt.q40) { + return (j.woff / 256l) * Q40_QSB + } elif (j.fmt == KqFmt.q51) { + return (j.woff / 32l) * Q51_QB + } + return j.woff +} + +//! Total bytes the streamed fill will append for `field` (page padding NOT included) — the +//! LAYOUT size when the sizing pass recorded one (holes included), else the job sum. +def stream_plane_bytes(t : Model; jobs : array; field : string) : int64 { + let lt = g_stream_plane_total?[field] ?? 0l + if (lt > 0l) { + return lt + } + var total = 0l + for (j in jobs) { + if (stream_field_of(t, j.fmt) == field) { + total += stream_job_shape(t, j).qb + } + } + return total +} + +// append `gap` zero bytes — a layout hole the jobs never cover (never-written plane space) +def private stream_zero_fill(gap : int64; var zeros : array; + append : block<(p : void?; nbytes : uint64) : void>) { + var left = gap + if (left > 0l && empty(zeros)) { + zeros |> resize(1048576) + } + while (left > 0l) { + let chunk = min(left, long_length(zeros)) + unsafe { + invoke(append, addr(zeros[0]), uint64(chunk)) + } + left -= chunk + } +} + +//! Execute every recorded job targeting `field`: transcode into a job-sized temp (load_big's +//! own arms at temp offset 0), land scales in the RAM plane, apply the job's repack regions +//! (temp-biased q, in-place s), hand the temp to `append`; qblob's tail runs the wscale epilogue. +def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; field : string; + jobs : array; regs : array; + append : block<(p : void?; nbytes : uint64) : void>) : bool { + // jobs for this field, ascending woff — plane order IS ascending source order + var mine : array + for (j in jobs) { + if (stream_field_of(t, j.fmt) == field) { + mine |> push(j) + } + } + if (empty(mine)) { + return true + } + mine |> sort() $(a, b : FillJob) => a.woff < b.woff + var temp_q : array + var temp_q8 : array // the q8 family's quant temp (qblob is int8) + var temp_s : array + var temp_sf : array + var scratch : array + var zeros : array + var cursor = 0l // plane bytes appended so far — layout holes between jobs zero-fill + for (j in mine) { + let sh = stream_job_shape(t, j) + stream_zero_fill(stream_job_qoff(j) - cursor, zeros, append) + cursor = stream_job_qoff(j) + sh.qb + temp_q |> resize(int(sh.qb)) + let t0 = g_conv_prof ? ref_time_ticks() : 0l + var kind = "q8 transcode (Q8_0 verbatim)" + if (j.fmt == KqFmt.k4) { + temp_s |> resize(int(sh.sb)) + gguf_transcode_q4k(m, bytes, j.name, temp_q, temp_s, 0l, j.n, j.src_off) + kind = "k4 transcode (Q4_K)" + } elif (j.fmt == KqFmt.k5) { + temp_s |> resize(int(sh.sb)) + if (gguf_tensor_type(m, j.name) == GGML_TYPE_Q5_K) { + gguf_transcode_q5k(m, bytes, j.name, temp_q, temp_s, 0l, j.n, j.src_off) + kind = "k5 transcode (Q5_K)" + } else { + scratch |> resize(int(j.n)) + gguf_read_tensor_f32(m, bytes, j.name, scratch, 0l, j.n, j.src_off) + quantize_k5_plane(scratch, j.n, temp_q, temp_s, 0l) + kind = "k5 RE-ENCODE (Q5_0 dequant+quant)" + } + } elif (j.fmt == KqFmt.k6) { + temp_s |> resize(int(sh.sb)) + gguf_transcode_q6k(m, bytes, j.name, temp_q, temp_s, 0l, j.n, j.src_off) + kind = "k6 transcode (Q6_K)" + } elif (j.fmt == KqFmt.q40) { + temp_s |> resize(int(sh.sb)) + gguf_transcode_q40(m, bytes, j.name, temp_q, temp_s, 0l, j.n, j.src_off) + kind = "q40 transcode (Q4_0)" + } elif (j.fmt == KqFmt.q51) { + temp_s |> resize(int(sh.sb)) + gguf_transcode_q51(m, bytes, j.name, temp_q, temp_s, 0l, j.n, j.src_off) + kind = "q51 transcode (Q5_1)" + } else { + // q8 family: scales are FLOATS; the epilogue converts the whole plane at the end + temp_q8 |> resize(int(j.n)) + temp_sf |> resize(int(j.n / 32l)) + let gt = gguf_tensor_type(m, j.name) + if (gt == GGML_TYPE_Q8_0) { + gguf_transcode_q8_0(m, bytes, j.name, temp_q8, temp_sf, 0l, 0l, j.n, j.src_off) + } elif (gt == GGML_TYPE_Q5_0) { + gguf_transcode_q5_0_to_q8(m, bytes, j.name, temp_q8, temp_sf, 0l, 0l, j.n, j.src_off) + kind = "q8 transcode (Q5_0 exact)" + } else { + scratch |> resize(int(j.n)) + gguf_read_tensor_f32(m, bytes, j.name, scratch, 0l, j.n, j.src_off) + quantize_q8_0_into(scratch, j.n, temp_q8, temp_sf, 0l, 0l) + kind = "q8 REQUANT (dequant+quant)" + } + } + conv_note(kind, t0, j.n) + // land the scale half at its real offset (RAM-resident plane) + unsafe { + if (j.fmt == KqFmt.k4) { + memcpy(addr(t.k4s[(j.woff / 256l) * K4_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.k5) { + memcpy(addr(t.k5s[(j.woff / 256l) * K5_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.k6) { + memcpy(addr(t.k6s[(j.woff / 256l) * K6_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.q40) { + memcpy(addr(t.q40s[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.q51) { + memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) + } elif (sh.sb == 0l && j.n > 0l) { + memcpy(addr(t.qscales[j.woff / 32l]), addr(temp_sf[0]), (j.n / 32l) * 4l) + } + } + // per-job repack: this job's regions applied on the temp (q half, pointer biased to + // woff 0) and the real scale plane (in place) — the same backend fns, same regions + unsafe { + for (r in regs) { + if (r.fmt != sh.rfmt || r.off < j.woff || r.off >= j.woff + j.n) { + continue + } + if (r.fmt == 0) { + invoke(g_stream_repack, 0, addr(temp_q8[r.off - j.woff]), + addr(t.qscales[r.off / 32l]), r.n, r.d) + } elif (r.fmt == 2) { + invoke(g_stream_repack, 2, addr(temp_q[((r.off - j.woff) / 32l) * Q51_QPB]), + addr(t.q51s[(r.off / 32l) * Q51_SPB]), r.n, r.d) + } elif (r.fmt == 4) { + invoke(g_stream_repack, 4, addr(temp_q[((r.off - j.woff) / 256l) * K4_QSB]), + addr(t.k4s[(r.off / 256l) * K4_SSB]), r.n, r.d) + } elif (r.fmt == 5) { + invoke(g_stream_repack, 5, addr(temp_q[((r.off - j.woff) / 256l) * K5_QSB]), + addr(t.k5s[(r.off / 256l) * K5_SSB]), r.n, r.d) + } elif (r.fmt == 6) { + invoke(g_stream_repack, 6, addr(temp_q[((r.off - j.woff) / 256l) * K6_QSB]), + addr(t.k6s[(r.off / 256l) * K6_SSB]), r.n, r.d) + } elif (r.fmt == 40) { + invoke(g_stream_repack, 40, addr(temp_q[((r.off - j.woff) / 256l) * Q40_QSB]), + addr(t.q40s[(r.off / 256l) * Q40_SSB]), r.n, r.d) + } + } + } + unsafe { + if (sh.rfmt == 0) { + invoke(append, addr(temp_q8[0]), uint64(sh.qb)) + } else { + invoke(append, addr(temp_q[0]), uint64(sh.qb)) + } + } + } + // the layout's tail past the last job (never-written space) completes the plane + stream_zero_fill(stream_plane_bytes(t, jobs, field) - cursor, zeros, append) + if (field == "qblob" && g_wscale_f16) { + // the epilogue: scales are complete and repacked; qscales16 lands BEFORE its field + // writes (qblob precedes qscales/qscales16 in Model field order) + wscale_convert_f16(t) + } + delete mine + delete temp_q + delete temp_q8 + delete temp_s + delete temp_sf + delete scratch + delete zeros + return true +} + +//! Arm plan recording: subsequent load_big calls append FillJobs instead of transcoding +//! (smalls still load eagerly — fblob is small and every flavor keeps it). +def fill_plan_begin { + g_fill_plan_mode = true + g_fill_plan |> clear() +} + +//! Disarm and move the recorded plan out (empty = nothing recorded). +def fill_plan_take(var out : array) { + g_fill_plan_mode = false + out <- g_fill_plan +} + def private load_big(m : GGUFMeta; bytes : array | #; name : string; var t : Model; woff, n : int64; var scratch : array; src_off : int64 = 0l; fmt : KqFmt = KqFmt.q8) { + if (g_fill_plan_mode) { + g_fill_plan |> push(FillJob(name = name, woff = woff, n = n, src_off = src_off, fmt = fmt)) + return + } let t0 = g_conv_prof ? ref_time_ticks() : 0l var kind = "f32 read (no quant)" if (fmt == KqFmt.k4) { @@ -1793,6 +2282,43 @@ def public get_kq_q51_native() : bool { return g_kq_q51_native } +// ===== DlimConfiguration sources (dasllama_config) ===== + +def private dlim_cpu_source_impl(var c : DlimCpuConfig) { + // the one sanctioned backend-select side effect — the identity itself is a pure formatter + select_matmul_backend_for_load() + c.backend = dlim_backend_name() // the cross-box bake's target alias when one is armed + c.q8_mr = active_q8_layout_mr() + c.q8_wbias = active_q8_wbias() + c.q8_kgroup = active_q8_kgroup() + c.kq_mr4 = active_kq_layout_mr(4) + c.kq_mr5 = active_kq_layout_mr(5) + c.kq_mr6 = active_kq_layout_mr(6) + c.kq_mr40 = active_kq_layout_mr(40) + c.q51_mr = active_q51_layout_mr() + // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) + c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() + c.kquant_native = g_kquant_native + c.kq_q40_native = g_kq_q40_native + c.kq_q50_native = g_kq_q50_native + c.kq_q51_native = g_kq_q51_native +} + +def private dlim_metal_source_impl(var c : DlimMetalConfig) : bool { + if (get_metal_mode() == MetalMode.off || !has_decode_override("metal")) { + return false + } + c.mode = "{get_metal_mode()}" + return true +} + +[init] +def dlim_config_sources_register() { + set_dlim_cpu_source(@@dlim_cpu_source_impl) + set_dlim_metal_source(@@dlim_metal_source_impl) + set_moe_gpu_slice_window_fn(@@vk_slice_window_provider) +} + def private count_fmt(a : array; f : KqFmt) : int { var n = 0 for (x in a) { @@ -2058,7 +2584,11 @@ def private moe_gpu_gather_upload(t : Model; f : KqFmt; woff : int64; n, rows, s : (f == KqFmt.k6 ? t.kq_repack_mr6 : t.kq_repack_mr40))) : 1l moe_gpu_gather_stack_kq(t, f, woff, n, rows, slice_rows, t.kq_repacked, kmr, wq, ws) } - return moe_gpu_upload_stack(wq, ws, woff, n, rows, int(f), stream) + let ok = moe_gpu_upload_stack(wq, ws, woff, n, rows, int(f), stream) + if (ok) { + vk_bake_collect(int(f), woff, n, rows, wq, ws) + } + return ok } //! What a whole-model resident load would cost on the device. The resident driver is all-or-nothing @@ -2188,6 +2718,9 @@ def private resident_place(t : Model; f : KqFmt; woff : int64; n, rows : int64; moe_gpu_gather_stack_kq(t, f, woff, n, rows, rows, t.kq_repacked, kmr, wq, ws) } let blk = rdec_place(wq, ws, n, rows, int(f)) + if (blk >= 0l) { + vk_bake_collect(int(f), woff, n, rows, wq, ws) + } delete wq delete ws return blk @@ -2222,6 +2755,7 @@ def private resident_layer_fmts(t : Model; l : int64) : tuple 0l) { return false } @@ -2764,13 +3298,15 @@ def moe_gpu_upload_resident(t : Model) { // DASLLAMA_GPU_CLS=0 keeps it on the CPU (A/B lever). let cls_fmt = t.config.shared_weights ? t.emb_fmt : t.wcls_fmt let cls_servable = cls_fmt == KqFmt.q8 ? (t.cls_q8 || !t.config.shared_weights) : moe_gpu_fmt_kq(cls_fmt) - if (cls_servable && env_flag("DASLLAMA_GPU_CLS", true) + vulkan_bake_role(VkBakeRole.cls) + if (cls_servable && gpu_want_cls_upload() && moe_gpu_gather_upload(t, cls_fmt, t.wcls_off, dim, t.config.vocab_size, t.config.vocab_size, mr, kgroup, wbias, wq, ws)) { set_moe_gpu_cls_off(t.wcls_off) } // dense (attention-side) planes second — DASLLAMA_GPU_DENSE=1 / gpu_dense opt-in. Arch-agnostic: // recurrent layers carry the deltanet triple, attention layers the q/o pair (k/v stay CPU — small). // Layer-ascending; each successful upload marks its plane, a budget stop ends the walk. + vulkan_bake_role(VkBakeRole.dense) if (gpu_want_dense()) { var dense_marked = 0l var dense_stop = false @@ -2819,6 +3355,7 @@ def moe_gpu_upload_resident(t : Model) { } // shared-expert triples — DASLLAMA_GPU_SHEXP=1 opt-in. Dense (every token, every layer, one // fixed weight set) so residency carries no miss term; q8 only (the repack registers them fmt 0). + vulkan_bake_role(VkBakeRole.shexp) if (gpu_want_shexp() && t.config.n_ff_shexp > 0l && !t.config.moe_dense_shexp) { let nsh = t.config.n_ff_shexp let she = dim * nsh @@ -2854,6 +3391,7 @@ def moe_gpu_upload_resident(t : Model) { // deltanet triples third — DASLLAMA_GPU_DN=1 opt-in: each recurrent layer's qkv/z-gate/ // out-proj planes + the layer mark that routes its whole prefill attention through the // device chain. Layer-ascending; a budget stop leaves later layers CPU, a partial triple rolls back. + vulkan_bake_role(VkBakeRole.dn) if (dn_want) { var dn_marked = 0l var dn_stop = false @@ -2897,6 +3435,7 @@ def moe_gpu_upload_resident(t : Model) { // attention quads fourth — DASLLAMA_GPU_ATTN=1 opt-in: each servable full-attention layer's // q|gate/k/v/wo planes + the layer mark that routes its whole prefill attention through the // device chain. Layer-ascending; a budget stop rolls back the partial quad. + vulkan_bake_role(VkBakeRole.attn) if (at_want) { let c = t.config // NEOX rope only (partial rotary implies NEOX per the loader); the chain has no bias / @@ -2961,6 +3500,7 @@ def moe_gpu_upload_resident(t : Model) { // attention q/k/v triples — DASLLAMA_GPU_QKV=1 opt-in. LAST of the dense rails ON PURPOSE: it // is the lowest-value one and an earlier section exhausting stacks/VRAM starves those behind it // (a 224-stack cap once cut 6 DN layers = -14% tg). Pays only where GPU reads beat CPU reads. + vulkan_bake_role(VkBakeRole.qkv) if (gpu_want_qkv()) { var qkv_marked = 0l for (l in range64(layers)) { @@ -2998,6 +3538,7 @@ def moe_gpu_upload_resident(t : Model) { to_log(LOG_INFO, "dasLLAMA: GPU MoE tier: q/k/v of {qkv_marked} attention layers resident\n") } } + vulkan_bake_role(VkBakeRole.expert) for (i in range64(want)) { let l = layers - 1l - i let f1 = fmt_at(t.we1_fmt, l) @@ -3036,6 +3577,7 @@ def moe_gpu_upload_resident(t : Model) { // streamed tail: DASLLAMA_GPU_MOE_STREAM=N marks the next N layers below the resident set for // prefill streaming — device-layout planes gathered into pinned host mirrors the tier DMAs per // prefill. CPU planes stay (decode runs them); the walk is contiguous descending. + vulkan_bake_role(VkBakeRole.expert_stream) var sfrom = from_l for (i in range64(has_experts ? max(nstream, 0l) : 0l)) { let l = from_l - 1l - i @@ -3115,6 +3657,93 @@ def load_gguf(path : string; mode : QuantMode = QuantMode.fp32) : Model { return <- t } +//! Streaming-conversion entry: plan-load `t`, then invoke `blk(m, bytes)` with the source +//! mapping STILL ALIVE — the caller streams the save inside. Fails open (unstreamable models +//! materialize + save eagerly through the same blk); split shards return false pre-blk. +def load_gguf_streaming(path : string; mode : QuantMode; var t : Model; var jobs : array; + blk : block<(m : GGUFMeta; bytes : array) : bool>) : bool { + if (!is_job_que_available()) { + var okj = false + with_job_que() { + okj = load_gguf_streaming(path, mode, t, jobs, blk) + } + return okj + } + g_weights_epoch++ + t.quant = mode + if (has_env_variable("DASLLAMA_CONV_PROF")) { + set_conv_profile(true) + } + let ts_load = ref_time_ticks() + let lmode0 = mode + var shards <- gguf_shard_paths(path) + if (length(shards) > 1) { + // split gguf: every shard mapping stays alive across the plan-load AND the blk-save + // (the transcode kernels resolve per-shard bases through the meta) + to_log(LOG_INFO, "dasLLAMA: split gguf — {length(shards)} shards (streaming)\n") + var sok = false + var bases : array + var sizes : array + gguf_map_shards(shards, 0, bases, sizes) { + if (env_flag("DASLLAMA_PREFETCH", true)) { + var pf_mb = 0l + for (bi in range(length(bases))) { + if (unsafe(prefetch_map(bases[bi], uint64(sizes[bi])))) { + pf_mb += sizes[bi] >> 20l + } + } + if (pf_mb > 0l) { + to_log(LOG_INFO, "dasLLAMA: source prefetch armed ({pf_mb} MB across {length(bases)} shards)\n") + } + } + var m <- parse_gguf_meta_shards(bases, sizes) + fill_plan_begin() + unsafe { + load_gguf_parsed(t, m, temp_array(reinterpret(bases[0]), sizes[0], type), lmode0, ts_load) + } + fill_plan_take(jobs) + t.tok <- load_tokenizer_auto(path) + unsafe { + sok = invoke(blk, m, temp_array(reinterpret(bases[0]), sizes[0], type)) + } + delete m + } + bases |> clear() // borrowed mapping bases — clear so delete frees only the buffer + delete bases + delete sizes + delete shards + return sok + } + delete shards + let f = fopen(path, "rb") + if (f == null) { + return false + } + var ok = false // nolint:LINT003 — written inside the fmap block + let lmode = mode + fmap(f) $(var bytes : array#) { + if (env_flag("DASLLAMA_PREFETCH", true)) { + var armed = false + unsafe { + armed = prefetch_map(addr(bytes[0]), uint64(long_length(bytes))) + } + if (armed) { + to_log(LOG_INFO, "dasLLAMA: source prefetch armed ({long_length(bytes) >> 20l} MB)\n") + } + } + var inscope m <- parse_gguf_meta(bytes) + fill_plan_begin() + load_gguf_parsed(t, m, bytes, lmode, ts_load) + fill_plan_take(jobs) + t.tok <- load_tokenizer_auto(path) + unsafe { + ok = invoke(blk, m, temp_array(addr(bytes[0]), long_length(bytes), type)) + } + } + fclose(f) + return ok +} + def private load_gguf_impl(path : string; var mode : QuantMode) : Model { g_weights_epoch++ var t = Model() @@ -3129,6 +3758,18 @@ def private load_gguf_impl(path : string; var mode : QuantMode) : Model { var bases : array var sizes : array gguf_map_shards(shards, 0, bases, sizes) { + // fault the source in AHEAD of the parallel transcode (in-loop faults serialize lanes) + if (env_flag("DASLLAMA_PREFETCH", true)) { + var pf_mb = 0l + for (bi in range(length(bases))) { + if (unsafe(prefetch_map(bases[bi], uint64(sizes[bi])))) { + pf_mb += sizes[bi] >> 20l + } + } + if (pf_mb > 0l) { + to_log(LOG_INFO, "dasLLAMA: source prefetch armed ({pf_mb} MB across {length(shards)} shards)\n") + } + } var m <- parse_gguf_meta_shards(bases, sizes) unsafe { load_gguf_parsed(t, m, temp_array(reinterpret(bases[0]), sizes[0], type), mode, ts_load) @@ -3145,6 +3786,15 @@ def private load_gguf_impl(path : string; var mode : QuantMode) : Model { panic("dasLLAMA: cannot open gguf '{path}'") } fmap(f) $(var bytes : array#) { + if (env_flag("DASLLAMA_PREFETCH", true)) { + var armed = false + unsafe { + armed = prefetch_map(addr(bytes[0]), uint64(long_length(bytes))) + } + if (armed) { + to_log(LOG_INFO, "dasLLAMA: source prefetch armed ({long_length(bytes) >> 20l} MB)\n") + } + } var inscope m <- parse_gguf_meta(bytes) load_gguf_parsed(t, m, bytes, mode, ts_load) } @@ -3641,6 +4291,13 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | && gguf_tensor_type(m, "per_layer_model_proj.weight") == GGML_TYPE_BF16) let kv = t.config.kv_dim let qd = t.config.n_heads * t.config.head_size // query width (== dim except Gemma2) + // streaming covers the q8-mode load_big rails; models with direct-write loaders + // (mx4 experts, grouped deltanet, bf16 PLE) disarm HERE — before sizing — so the + // eager path proceeds with fully materialized planes and an empty job list + if (g_fill_plan_mode && (mode != QuantMode.q8 || t.experts_mx4 || t.dn_grouped || t.ple_model_bf16)) { + to_log(LOG_INFO, "dasLLAMA: streaming plan disarmed — unstreamed rail (mode {mode}, mx4 {t.experts_mx4}, dn-grouped {t.dn_grouped}, bf16-ple {t.ple_model_bf16}); converting eagerly\n") + g_fill_plan_mode = false + } let sz = layout_offsets(t) // fp32 aux (embedding + norms) is always present; the big-weight storage matches `mode` reserve_resize(t.fblob, sz.fblob_n) @@ -3654,39 +4311,64 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.mxs |> reserve(sz.mx_n / 32l) t.mxs |> resize(sz.mx_n / 32l) } + // streaming: big q-halves never materialize (jobs stream them to the writer); scale + // halves are small (6-16%) and stay in RAM. Unstreamable features disarmed above. + let stream_q = g_fill_plan_mode + if (stream_q) { + // the streamed save sizes each q-plane from the LAYOUT (holes included), not the job sum + g_stream_plane_total |> clear() + g_stream_plane_total["k4q"] = (sz.k4_n / 256l) * K4_QSB + g_stream_plane_total["k5q"] = (sz.k5_n / 256l) * K5_QSB + g_stream_plane_total["k6q"] = (sz.k6_n / 256l) * K6_QSB + g_stream_plane_total["q40q"] = (sz.q40_n / 256l) * Q40_QSB + g_stream_plane_total["q51q"] = (sz.q51_n / 32l) * Q51_QB + g_stream_plane_total["qblob"] = mode == QuantMode.q8 ? sz.wblob_n : 0l + } if (sz.k4_n > 0l) { - t.k4q |> reserve((sz.k4_n / 256l) * K4_QSB) - t.k4q |> resize((sz.k4_n / 256l) * K4_QSB) + if (!stream_q) { + t.k4q |> reserve((sz.k4_n / 256l) * K4_QSB) + t.k4q |> resize((sz.k4_n / 256l) * K4_QSB) + } t.k4s |> reserve((sz.k4_n / 256l) * K4_SSB) t.k4s |> resize((sz.k4_n / 256l) * K4_SSB) } if (sz.k5_n > 0l) { - t.k5q |> reserve((sz.k5_n / 256l) * K5_QSB) - t.k5q |> resize((sz.k5_n / 256l) * K5_QSB) + if (!stream_q) { + t.k5q |> reserve((sz.k5_n / 256l) * K5_QSB) + t.k5q |> resize((sz.k5_n / 256l) * K5_QSB) + } t.k5s |> reserve((sz.k5_n / 256l) * K5_SSB) t.k5s |> resize((sz.k5_n / 256l) * K5_SSB) } if (sz.k6_n > 0l) { - t.k6q |> reserve((sz.k6_n / 256l) * K6_QSB) - t.k6q |> resize((sz.k6_n / 256l) * K6_QSB) + if (!stream_q) { + t.k6q |> reserve((sz.k6_n / 256l) * K6_QSB) + t.k6q |> resize((sz.k6_n / 256l) * K6_QSB) + } t.k6s |> reserve((sz.k6_n / 256l) * K6_SSB) t.k6s |> resize((sz.k6_n / 256l) * K6_SSB) } if (sz.q40_n > 0l) { - t.q40q |> reserve((sz.q40_n / 256l) * Q40_QSB) - t.q40q |> resize((sz.q40_n / 256l) * Q40_QSB) + if (!stream_q) { + t.q40q |> reserve((sz.q40_n / 256l) * Q40_QSB) + t.q40q |> resize((sz.q40_n / 256l) * Q40_QSB) + } t.q40s |> reserve((sz.q40_n / 256l) * Q40_SSB) t.q40s |> resize((sz.q40_n / 256l) * Q40_SSB) } if (sz.q51_n > 0l) { - t.q51q |> reserve((sz.q51_n / 32l) * Q51_QB) - t.q51q |> resize((sz.q51_n / 32l) * Q51_QB) + if (!stream_q) { + t.q51q |> reserve((sz.q51_n / 32l) * Q51_QB) + t.q51q |> resize((sz.q51_n / 32l) * Q51_QB) + } t.q51s |> reserve((sz.q51_n / 32l) * Q51_SB) t.q51s |> resize((sz.q51_n / 32l) * Q51_SB) } if (mode == QuantMode.q8) { - t.qblob |> reserve(sz.wblob_n) - t.qblob |> resize(sz.wblob_n) + if (!stream_q) { + t.qblob |> reserve(sz.wblob_n) + t.qblob |> resize(sz.wblob_n) + } t.qscales |> reserve(sz.wblob_n / 32l) t.qscales |> resize(sz.wblob_n / 32l) } elif (mode == QuantMode.q4_0) { @@ -3922,9 +4604,10 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | if (!t.config.shared_weights) { load_big(m, bytes, "output.weight", t, t.wcls_off, vocab * dim, scratch, 0l, t.wcls_fmt) } elif (t.cls_q8) { - // tied Q8: classifier copy (repacked below) + the linear copy embedding rows read - gguf_transcode_q8_0(m, bytes, "token_embd.weight", t.qblob, t.qscales, t.wcls_off, t.wcls_off / 32l, vocab * dim) - gguf_transcode_q8_0(m, bytes, "token_embd.weight", t.qblob, t.qscales, t.emb_q8_off, t.emb_q8_off / 32l, vocab * dim) + // tied Q8: classifier copy (repacked below) + the linear copy embedding rows read. + // Through load_big (its q8 arm is the same transcode) so the streaming plan records both. + load_big(m, bytes, "token_embd.weight", t, t.wcls_off, vocab * dim, scratch) + load_big(m, bytes, "token_embd.weight", t, t.emb_q8_off, vocab * dim, scratch) } elif (cls_kq(t)) { // tied K-quant: ONE plane copy serves the classifier matmul and embedding row reads // (the kq planes stay in disk order — no repack, so no second linear copy) @@ -3955,31 +4638,53 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | // arm64: switch to the laneq backend and repack the just-loaded Q8 weights into its interleaved // layout. Returns false off-ARM (portable stays, weights stay row-major). if (mode == QuantMode.q8 && select_matmul_backend_for_load()) { - repack_q8_weights(t) - if (t.experts_mx4) { - repack_mx4_stacks(t) - } - if (active_q51_layout_mr() > 1l) { - repack_q51_stacks(t) - } - if (t.kquant_native && kernel_backend_has_kq()) { - repack_kq_weights(t) - t.kq_repacked = true - t.kq_repack_mr4 = active_kq_layout_mr(4) - t.kq_repack_mr5 = active_kq_layout_mr(5) - t.kq_repack_mr6 = active_kq_layout_mr(6) - t.kq_repack_mr40 = active_kq_layout_mr(40) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40})\n") + if (g_fill_plan_mode) { + // streaming: the q-halves are not materialized — the fill applies the SAME + // regions per job. Set the marks the eager walkers would (the meta serializes + // BEFORE the streamed planes fill), run nothing. + t.mx4_repacked = true + if (t.kquant_native && kernel_backend_has_kq()) { + t.kq_repacked = true + t.kq_repack_mr4 = active_kq_layout_mr(4) + t.kq_repack_mr5 = active_kq_layout_mr(5) + t.kq_repack_mr6 = active_kq_layout_mr(6) + t.kq_repack_mr40 = active_kq_layout_mr(40) + } + } else { + repack_q8_weights(t) + if (t.experts_mx4) { + repack_mx4_stacks(t) + } + if (active_q51_layout_mr() > 1l) { + repack_q51_stacks(t) + } + if (t.kquant_native && kernel_backend_has_kq()) { + repack_kq_weights(t) + t.kq_repacked = true + t.kq_repack_mr4 = active_kq_layout_mr(4) + t.kq_repack_mr5 = active_kq_layout_mr(5) + t.kq_repack_mr6 = active_kq_layout_mr(6) + t.kq_repack_mr40 = active_kq_layout_mr(40) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40})\n") + } } } to_log(LOG_INFO, "dasLLAMA: load stage repack: {get_time_usec(ts_stage) / 1000} ms\n") ts_stage = ref_time_ticks() if (mode == QuantMode.q8 && g_wscale_f16) { - wscale_convert_f16(t) + if (g_fill_plan_mode) { + // streaming: the epilogue converts after qblob fills; pre-mark for the meta, + // mirroring the convert's own outcome (empty scales = nothing converts, no flag) + t.wscale_f16 = !empty(t.qscales) && kernel_backend_has_wscale16() + } else { + wscale_convert_f16(t) + } } to_log(LOG_INFO, "dasLLAMA: load stage wscale: {get_time_usec(ts_stage) / 1000} ms, total {get_time_usec(ts_load) / 1000} ms\n") log_load_report(t) - moe_gpu_upload_resident(t) + if (!g_fill_plan_mode) { // streaming converts only — no serving, no gathers over unmade planes + moe_gpu_upload_resident(t) + } } } @@ -8809,13 +9514,33 @@ def private dequant_q8_row(t : Model; off, n : int64; var dst : float?) { } } +// trimmed-flavor q8 embedding read: embq holds the emb region as mblob-style 34B interleaved +// blocks (f16 scale + 32 int8), region-rebased — same loop as dequant_q8_row's mblob arm +def private dequant_embq_row(t : Model; off, n : int64; var dst : float?) { + unsafe { + let bp = addr < uint8 const? >(t.embq[(off / 32l) * 34l]) + for (b in range64(n / 32l)) { + let sc = f16_to_f32(uint(bp[b * 34l]) | (uint(bp[b * 34l + 1l]) << 8u)) + let qp = reinterpret(bp + b * 34l + 2l) + let base = b * 32l + for (k in range64(32l)) { + dst[base + k] = float(qp[k]) * sc + } + } + } +} + // Copy token `token`'s embedding row into dst — from fp32 fblob, or (cls_q8) dequanted from the -// linear Q8 copy at emb_q8_off, or (cls_kq) from the K-quant plane at wcls_off. Rows are whole -// superblocks (dim % 256 == 0, a ggml K-quant invariant). +// linear Q8 copy at emb_q8_off (embq when trimmed), or (cls_kq) from the K-quant plane at +// wcls_off (embq/embs when trimmed). Rows are whole superblocks (a ggml K-quant invariant). def private embed_row(t : Model; token : int64; var dst : float?) { let dim = t.config.dim if (t.cls_q8) { - dequant_q8_row(t, t.emb_q8_off + token * dim, dim, dst) + if (t.planes_trimmed) { + dequant_embq_row(t, token * dim, dim, dst) + } else { + dequant_q8_row(t, t.emb_q8_off + token * dim, dim, dst) + } } elif (cls_kq(t)) { let mr = (t.emb_fmt == KqFmt.k4 ? t.kq_repack_mr4 : (t.emb_fmt == KqFmt.k5 ? t.kq_repack_mr5 @@ -8826,15 +9551,29 @@ def private embed_row(t : Model; token : int64; var dst : float?) { let nsb = dim / 256l let g = token / mr let fmt = int64(kq_fi(t.emb_fmt)) - let sb0t = t.wcls_off / 256l - if (t.emb_fmt == KqFmt.k4) { - dequant_kq_row_grp(fmt, addr(t.k4q[(sb0t + g * mr * nsb) * K4_QSB]), addr(t.k4s[(sb0t + g * mr * nsb) * K4_SSB]), token % mr, mr, dim, dst) + // trimmed: embq/embs hold the region's plane pair verbatim, rebased to 0 + let sb0t = t.planes_trimmed ? 0l : t.wcls_off / 256l + let sbg = (sb0t + g * mr * nsb) + if (t.planes_trimmed) { + if (t.emb_fmt == KqFmt.k4) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K4_QSB]), addr(t.embs[sbg * K4_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k5) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K5_QSB]), addr(t.embs[sbg * K5_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k6) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K6_QSB]), addr(t.embs[sbg * K6_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.q40) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } else { + panic("embed_row: '{t.emb_fmt}' has no kq plane pair") + } + } elif (t.emb_fmt == KqFmt.k4) { + dequant_kq_row_grp(fmt, addr(t.k4q[sbg * K4_QSB]), addr(t.k4s[sbg * K4_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.k5) { - dequant_kq_row_grp(fmt, addr(t.k5q[(sb0t + g * mr * nsb) * K5_QSB]), addr(t.k5s[(sb0t + g * mr * nsb) * K5_SSB]), token % mr, mr, dim, dst) + dequant_kq_row_grp(fmt, addr(t.k5q[sbg * K5_QSB]), addr(t.k5s[sbg * K5_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.k6) { - dequant_kq_row_grp(fmt, addr(t.k6q[(sb0t + g * mr * nsb) * K6_QSB]), addr(t.k6s[(sb0t + g * mr * nsb) * K6_SSB]), token % mr, mr, dim, dst) + dequant_kq_row_grp(fmt, addr(t.k6q[sbg * K6_QSB]), addr(t.k6s[sbg * K6_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.q40) { - dequant_kq_row_grp(fmt, addr(t.q40q[(sb0t + g * mr * nsb) * Q40_QSB]), addr(t.q40s[(sb0t + g * mr * nsb) * Q40_SSB]), token % mr, mr, dim, dst) + dequant_kq_row_grp(fmt, addr(t.q40q[sbg * Q40_QSB]), addr(t.q40s[sbg * Q40_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -8842,12 +9581,25 @@ def private embed_row(t : Model; token : int64; var dst : float?) { } else { unsafe { var row = temp_array(dst, dim, type) - let sb0 = (t.wcls_off + token * dim) / 256l + // trimmed: embq/embs hold the region's plane pair verbatim, rebased to 0 + let sb0 = t.planes_trimmed ? (token * dim) / 256l : (t.wcls_off + token * dim) / 256l // metal-blob planes: k4/k5 strips are 16B, k6 is the split GPU form (d in the tail) let ssb45 = t.metal_blob ? 16l : K4_SSB let k6d0 = t.metal_blob ? (long_length(t.k6s) / K6_SSB) * 16l : 0l for (s in range64(dim / 256l)) { - if (t.emb_fmt == KqFmt.k4) { + if (t.planes_trimmed) { + if (t.emb_fmt == KqFmt.k4) { + dequant_k4_plane_superblock(t.embq, (sb0 + s) * K4_QSB, t.embs, (sb0 + s) * K4_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k5) { + dequant_k5_plane_superblock(t.embq, (sb0 + s) * K5_QSB, t.embs, (sb0 + s) * K5_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k6) { + dequant_k6_plane_superblock(t.embq, (sb0 + s) * K6_QSB, t.embs, (sb0 + s) * K6_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.q40) { + dequant_q40_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) + } else { + panic("embed_row: '{t.emb_fmt}' has no kq plane pair") + } + } elif (t.emb_fmt == KqFmt.k4) { dequant_k4_plane_superblock(t.k4q, (sb0 + s) * K4_QSB, t.k4s, (sb0 + s) * ssb45, row, s * 256l) } elif (t.emb_fmt == KqFmt.k5) { dequant_k5_plane_superblock(t.k5q, (sb0 + s) * K5_QSB, t.k5s, (sb0 + s) * ssb45, row, s * 256l) @@ -8876,6 +9628,10 @@ def private embed_row(t : Model; token : int64; var dst : float?) { def private mm_cls(var y : array; t : Model; x : array; var xq : array; var xs : array; var xbs : array; n, d : int64) { let cls_fmt = t.config.shared_weights ? t.emb_fmt : t.wcls_fmt let tied_f32 = t.config.shared_weights && !t.cls_q8 && cls_fmt == KqFmt.q8 + if (t.planes_trimmed && !moe_gpu_cls_on_gpu(t.wcls_off)) { + // fail closed LOUDLY: the CPU classifier path reads the trimmed families + panic("dasLLAMA: planes_trimmed image — the classifier must serve on the GPU (rebake without trim, or arm the tier as the image's config says)") + } if (moe_gpu_cls_on_gpu(t.wcls_off)) { if (cls_fmt == KqFmt.q8) { quantize_q8_0_into(x, n, xq, xs, 0l, 0l) diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das new file mode 100644 index 0000000000..35c773874f --- /dev/null +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -0,0 +1,144 @@ +options gen2 +options _comment_hygiene = true + +//! The explicit .dlim bake configuration: every input that changes image BYTES, in one struct — +//! built at load time from the live box, suppliable to the converter as JSON, embedded in the +//! image meta. The identity is a pure formatter of this struct; nothing in here probes the host. +//! +//! Discipline rule: a knob that changes image bytes MUST be a field here (missing = silent stale +//! images); a serve-only knob MUST NOT be (present = spurious rebakes on every knob flip). + +module dasllama_config shared public + +require strings +require daslib/utf8_utils + +//! The CPU tier's layout half: which backend's repack shapes the planes, its interleave numbers, +//! and the scale/native-format outcomes. `wscale_f16` is the OUTCOME (backend has the s16 twins), +//! not the request — an image's identity must describe its bytes, not its intent. +struct public DlimCpuConfig { + backend : string + q8_mr : int64 + q8_wbias : int64 + q8_kgroup : int64 + kq_mr4 : int64 + kq_mr5 : int64 + kq_mr6 : int64 + kq_mr40 : int64 + q51_mr : int64 + wscale_f16 : bool + kquant_native : bool + kq_q40_native : bool + kq_q50_native : bool + kq_q51_native : bool +} + +//! The vulkan flavor's plan inputs, RESOLVED — `coopmat_mode` is the running mode (not the raw +//! env string), `vram_mb` the concrete budget the plan is computed against. The dense bits and +//! `cls` are here so every walk-shaping switch is in the identity (the old tag missed them). +struct public DlimVulkanConfig { + coopmat_mode : int + expert_q8n : bool + vram_mb : int64 + moe_layers : int64 + moe_stream : int64 + dn : bool + attn : bool + dnd : bool + shexp : bool + qkv : bool + cls : bool + dense : bool + dense_attn : bool + dense_shexp : bool + trim : bool //!< P3: big CPU weight families dropped from the image (GPU-resident consumers only) + subgroup_size : int + max_storage_range : int64 +} + +//! The metal flavor's half. Minimal for now — the metal-flavor audit (arc phase G) decides +//! whether servability needs more than the mode. +struct public DlimMetalConfig { + mode : string +} + +//! The whole bake configuration. `vulkan_on`/`metal_on` are presence flags — a flavor section +//! whose tier is absent on this box stays zeroed and MUST NOT be read. +struct public DlimConfiguration { + quant : string = "q8" + cpu : DlimCpuConfig + vulkan_on : bool + vulkan : DlimVulkanConfig + metal_on : bool + metal : DlimMetalConfig +} + +// ===== section sources (tiers register at [init]; config never probes the host itself) ===== + +[unused_argument(c)] +def private dlim_no_cpu_source(var c : DlimCpuConfig) { + panic("dasLLAMA: no DlimConfiguration cpu source registered (dasllama_common's [init] provides it)") +} + +[unused_argument(c)] +def private dlim_no_vulkan_source(var c : DlimVulkanConfig) : bool => false + +[unused_argument(c)] +def private dlim_no_metal_source(var c : DlimMetalConfig) : bool => false + +var private g_dlim_cpu_source = @@dlim_no_cpu_source +var private g_dlim_vulkan_source = @@dlim_no_vulkan_source +var private g_dlim_metal_source = @@dlim_no_metal_source + +//! The CPU tier registers its section filler (must run its backend select ONCE before reading +//! the layout getters — that side effect lives in the source, never in the identity). +def public set_dlim_cpu_source(f : function<(var c : DlimCpuConfig) : void>) { + g_dlim_cpu_source = f +} + +//! A GPU tier registers its section filler; returns false when the tier is absent/unarmed. +def public set_dlim_vulkan_source(f : function<(var c : DlimVulkanConfig) : bool>) { + g_dlim_vulkan_source = f +} + +def public set_dlim_metal_source(f : function<(var c : DlimMetalConfig) : bool>) { + g_dlim_metal_source = f +} + +//! Build the configuration this process would bake/load with. The ONE place host state is read. +def public dlim_config_current(quant : string = "q8") : DlimConfiguration { + var c = DlimConfiguration(quant = quant) + invoke(g_dlim_cpu_source, c.cpu) + c.vulkan_on = invoke(g_dlim_vulkan_source, c.vulkan) + c.metal_on = invoke(g_dlim_metal_source, c.metal) + return c +} + +// ===== pure formatters (identity, tags, JSON) ===== + +//! The identity string an image file/header is keyed by — a pure formatter of the struct. +//! `image_version` comes from the caller (dasllama_image owns the version constant). +def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { + return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}" + + "|q51 mr{c.cpu.q51_mr}" + + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + + (tag != "" ? "|{tag}" : "")) +} + +//! The vulkan flavor's identity tag, from the RESOLVED section (the old tag folded raw env +//! strings, so "auto" hid device-derived byte differences). +def public dlim_vulkan_tag(v : DlimVulkanConfig) : string { + return ("vulkan|m{v.coopmat_mode}{v.expert_q8n ? "n" : ""} b{v.vram_mb}" + + " l{v.moe_layers} s{v.moe_stream}" + + " r{v.dn ? 1 : 0}{v.attn ? 1 : 0}{v.dnd ? 1 : 0}{v.shexp ? 1 : 0}{v.qkv ? 1 : 0}{v.cls ? 1 : 0}" + + "{v.dense ? 1 : 0}{v.dense_attn ? 1 : 0}{v.dense_shexp ? 1 : 0}{v.trim ? " T" : ""}") +} + +//! Round-trip: the JSON embedded in the image meta and consumed by `dasllama-convert --config`. +def public dlim_config_json(c : DlimConfiguration) : string => sprint_json(c, true) + +// hand-carried config files grow a UTF-8 BOM on Windows editors/tools — tolerate it +def public dlim_config_from_json(text : string; var c : DlimConfiguration) : bool => ( + sscan_json(contains_utf8_bom(text) ? slice(text, 3) : text, c)) diff --git a/modules/dasLLAMA/dasllama/dasllama_env.das b/modules/dasLLAMA/dasllama/dasllama_env.das index 0609e1ac90..a285b54a1c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_env.das +++ b/modules/dasLLAMA/dasllama/dasllama_env.das @@ -229,6 +229,12 @@ def private reg_vulkan(var o : array) { "Fused decode tail (add+rms+requant, qk-norm+rope); 0 pins the split dispatches for a same-build A/B.") k(o, "DASLLAMA_VK_XFERQ", EnvKind.flag, EnvArea.vulkan, "on", "Stream expert uploads on the dedicated transfer queue, overlapped via a timeline semaphore; 0 keeps the single-queue rail.") + k(o, "DASLLAMA_PREFETCH", EnvKind.flag, EnvArea.engine, "on", + "Advisory source-mapping readahead at gguf load (cold-conversion fix); =0 restores on-demand faulting.") + k(o, "DASLLAMA_VK_IMPORT", EnvKind.flag, EnvArea.vulkan, "on", + "Stream mirrors import the mapped .dlim (VK_EXT_external_memory_host) instead of pinned copies; =0 restores the copy path.") + k(o, "DASLLAMA_TRIM", EnvKind.flag, EnvArea.vulkan, "off", + "Serve from P3-trimmed vulkan images (big CPU weight families dropped; folded into the flavor identity).") k(o, "DASLLAMA_VK_MEMPRIO", EnvKind.flag, EnvArea.vulkan, "on", "Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours.") k(o, "DASLLAMA_VK_FA", EnvKind.flag, EnvArea.vulkan, "on", @@ -372,6 +378,8 @@ def private reg_test(var o : array) { "Directory of audio corpus files for the transcription tests.") k(o, "TMPDIR", EnvKind.path, EnvArea.test, "/tmp", "Scratch directory for test artifacts; set by the OS on macOS.") + k(o, "TEMP", EnvKind.path, EnvArea.test, "", + "Windows scratch-directory fallback when TMPDIR is unset (the test runner's log dir).") } def private reg_example(var o : array) { diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 6c0076b519..68174a3937 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -13,6 +13,8 @@ require daslib/array_boost // nolint:STYLE030 — temp_array is [template]; ST require dasllama/dasllama_common require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor image transform require dasllama/dasllama_math // active_kernel_backend — half the image identity +require dasllama/dasllama_config // DlimConfiguration + dlim_identity — the identity formatter +require math // The prepared-model image (spec: fmap-model-image plan, locked 2026-07-16): the post-load // Model dumped once — planes POST-repack, page-aligned — then mapped back with zero O(model) @@ -31,7 +33,7 @@ require dasllama/dasllama_math // active_kernel_backend — half the image ide // config-derived. A loader change that alters which tensors load leaves a structurally VALID stale // image that silently represents a different model; only this version catches it. Mismatch // regenerates from the gguf, never migrates. -let IMAGE_VERSION = 6 // v6: glm4moe NextN heads now load (blk.n_layers + its qkv biases) +let IMAGE_VERSION = 7 // v7: identity string + DlimConfiguration JSON ride the meta head (readable declines; the .dlim-arc format changes batch into this one bump) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -48,23 +50,151 @@ struct ImgSection { bytes : uint64 } -// the box + knob identity an image is valid for; folded into filename AND header. Must be computed -// AFTER the same backend select the gguf loader runs at repack time (entry-time default differs). -// `tag` distinguishes image kinds sharing one source path (e.g. "tower-f32" vs "tower-q8"). -def image_identity(tag : string = "") : string { - select_matmul_backend_for_load() - return ("v{IMAGE_VERSION}|{active_kernel_backend()}|{get_wscale_f16() ? "s16" : "s32"}" - + "|q8 mr{active_q8_layout_mr()} b{active_q8_wbias()} g{active_q8_kgroup()}" - + "|kq {active_kq_layout_mr(4)}/{active_kq_layout_mr(5)}/{active_kq_layout_mr(6)}/{active_kq_layout_mr(40)}" - + "|q51 mr{active_q51_layout_mr()}" - + (tag != "" ? "|{tag}" : "")) +// the box + knob identity an image is valid for; folded into filename AND header. A PURE +// formatter of DlimConfiguration — the backend-select side effect lives in the config's cpu +// source, so callers need no ordering ritual. `tag` distinguishes kinds sharing one source path. +def image_identity(tag : string = ""; quant : string = "q8") : string { + return dlim_identity(dlim_config_current(quant), IMAGE_VERSION, tag) } // the image path for (gguf, current identity) — knob combos coexist as separate files // (the full 64-bit hash keeps that deterministic in practice; the header check would catch // a collision loudly, but two identities fighting over one path re-save forever) -def image_path_for(gguf_path : string; tag : string = "") : string { - return "{gguf_path}.{hash(image_identity(tag))}.dlim" +def image_path_for(gguf_path : string; tag : string = ""; quant : string = "q8") : string { + return "{gguf_path}.{hash(image_identity(tag, quant))}.dlim" +} + +//! What list/GC tooling reads from an image WITHOUT loading it: the header fields plus the +//! meta head's baked identity and embedded config JSON (empty when the version's layout is +//! unknown — pre-v7 images carry no readable head). +struct DlimPeek { + valid : bool //!< magic/page/bounds all sane (any version) + version : int + bytes : int64 + identity : string + config_json : string +} + +def image_peek(path : string) : DlimPeek { + var p = DlimPeek() + var msize = 0ul + var base : void? + unsafe { + base = fmap_open(path, addr(msize)) + } + if (base == null || int64(msize) < IMAGE_HEADER_BYTES) { + if (base != null) { + unsafe(fmap_close(base, msize)) + } + return p + } + var bp = unsafe(reinterpret(base)) + p.bytes = int64(msize) + p.version = int(read_u32(bp, 4l)) + let meta_off = read_u64(bp, 16l) + let meta_bytes = read_u64(bp, 24l) + p.valid = (read_u32(bp, 0l) == IMAGE_MAGIC && read_u32(bp, 8l) == uint(IMAGE_PAGE) + && read_u64(bp, 32l) == msize && meta_bytes != 0ul && meta_bytes <= msize + && meta_off <= msize - meta_bytes && meta_bytes <= uint64(INT_MAX)) + if (p.valid && p.version >= 7) { + var head : array + head |> resize(int(min(meta_bytes, 65536ul))) + unsafe { + memcpy(addr(head[0]), reinterpret(bp + int64(meta_off)), uint64(long_length(head))) + } + var hmeta <- new MemSerializer(head) + var harch = Archive(reading = true, stream = hmeta) + harch |> serialize(p.identity) + harch |> serialize(p.config_json) + if (!hmeta->OK()) { + p.identity = "" + p.config_json = "" + } + delete head + unsafe { + delete hmeta + } + } + unsafe(fmap_close(base, msize)) + return p +} + +//! One .dlim sibling of a GGUF, as list/GC tooling sees it (converter --list, the server page). +struct DlimImageInfo { + file : string //!< basename + path : string + bytes : int64 + version : int + verdict : string //!< CURRENT | OTHER | STALE vN | BROKEN + identity : string +} + +//! The identities THIS box would produce right now, across flavors and quants — the CURRENT +//! verdict's reference set. Callers pin the box profile first (same ordering as the load path). +def dlim_current_identities : table { + var idents : table + for (q in ["q8", "fp32", "q4_0"]) { + idents |> insert(image_identity("", q)) + idents |> insert(image_identity(METAL_IMAGE_TAG, q)) + let vk = moe_gpu_bake_tag() + if (vk != "") { + idents |> insert(image_identity(vk, q)) + } + } + return <- idents +} + +def dlim_image_verdict(p : DlimPeek; current : table) : string { + if (!p.valid) { + return "BROKEN" + } + if (p.version != IMAGE_VERSION) { + return "STALE v{p.version}" + } + return key_exists(current, p.identity) ? "CURRENT" : "OTHER" +} + +//! Every .dlim beside `gguf_path` (or in a directory), with sizes and verdicts. +def dlim_inventory(gguf_path : string) : array { + var out : array + var st : FStat + if (!stat(gguf_path, st)) { + return <- out + } + let dirp = st.is_dir ? gguf_path : dir_name(gguf_path) + let base = st.is_dir ? "" : base_name(gguf_path) + var current <- dlim_current_identities() + dir(dirp) $(name : string) { + if (!(name |> ends_with(".dlim")) || (base != "" && !(name |> starts_with(base)))) { + return + } + let full = path_join(dirp, name) + let p = image_peek(full) + out |> push(DlimImageInfo(file = name, path = full, bytes = p.bytes, version = p.version, + verdict = dlim_image_verdict(p, current), identity = p.identity)) + } + delete current + return <- out +} + +//! The dlim GC: remove what can never load again (BROKEN + version-stale) beside `gguf_path`. +//! NEVER a loadable image — OTHER may be another box's bake carried here on purpose. +//! apply = false is a dry run (stale counted, nothing removed). +def dlim_clean(gguf_path : string; apply : bool) : tuple { + var r = (total = 0, stale = 0, removed = 0, freed = 0l) + var inv <- dlim_inventory(gguf_path) + for (im in inv) { + r.total++ + continue if (im.verdict != "BROKEN" && !(im.verdict |> starts_with("STALE"))) + r.stale++ + if (apply && remove(im.path)) { + r.removed++ + r.freed += im.bytes + to_log(LOG_INFO, "dasLLAMA image: GC removed {im.file} ({im.bytes >> 20l} MB, {im.verdict})\n") + } + } + delete inv + return r } def private pad_to_page(cur : uint64) : uint64 { @@ -72,16 +202,84 @@ def private pad_to_page(cur : uint64) : uint64 { return (cur + page - 1ul) / page * page } -def private write_zeros(f : file; n : uint64) { - if (n == 0ul) { +let private IMAGE_WRITE_BAND = 16l * 1024l * 1024l // the direct writer's staging band +let private SECTION_TABLE_SLACK = 65536ul // preallocation headroom for the section blob + +// The image's byte sink: append-only and strictly ascending, which is what keeps NTFS from +// charging valid-data-length zero-fill on the preallocated file (measured: 3.4 s per 8 GB gap). +struct private ImgWriter { + h : void? = null + cur : uint64 = 0ul + ok : bool = true +} + +def private w_open(path : string; total : uint64) : ImgWriter { + let h = unsafe(dwrite_open(path, total, uint64(IMAGE_WRITE_BAND))) + return ImgWriter(h = h, ok = h != null) +} + +def private w_append(var w : ImgWriter; p : void?; nbytes : uint64) { + if (!w.ok || nbytes == 0ul) { + return + } + if (!unsafe(dwrite_append(w.h, p, nbytes))) { + w.ok = false // a dead save stays dead — don't keep pushing bytes at a full disk + return + } + w.cur += nbytes +} + +def private w_zeros(var w : ImgWriter; n : uint64) { + if (n == 0ul || !w.ok) { return } var z : array z |> resize(int(n)) - f |> fwrite(z) + unsafe { + w_append(w, addr(z[0]), n) + } delete z } +def private w_close(var w : ImgWriter) : bool { + if (w.h == null) { + return false + } + let flushed = unsafe(dwrite_close(w.h)) + w.h = null + return flushed && w.ok +} + +def private plane_end(cur : uint64; field : array) : uint64 { + if (empty(field)) { + return cur + } + return pad_to_page(cur) + uint64(long_length(field)) * uint64(typeinfo sizeof(field[0])) +} + +// The image's final size, known before a byte goes out: planes contribute their sizes, the meta +// blob is already serialized. The writer preallocates from this, so the file lands as one +// contiguous run. +def private image_total_bytes(var t; meta_bytes : uint64) : uint64 { + var cur = uint64(IMAGE_PAGE) + apply(t) $ [unused_argument(name)] (name : string; var field) { + static_if (typeinfo is_array(field)) { + static_if (!typeinfo is_string(field[0])) { + cur = plane_end(cur, field) + } + } static_elif (typeinfo safe_has_field < image_map > (field)) { + apply(field) $ [unused_argument(iname)] (iname : string; var ifield) { + static_if (typeinfo is_array(ifield)) { + static_if (!typeinfo is_string(ifield[0])) { + cur = plane_end(cur, ifield) + } + } + } + } + } + return pad_to_page(cur + meta_bytes) +} + def private store_u32(var b : array; at : int; v : uint) { unsafe { var p = unsafe(addr < uint? >(b[at])) @@ -246,6 +444,7 @@ def private serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.ple_post_off) arch |> serialize_raw(t.wscale_f16) arch |> serialize_raw(t.metal_blob) + arch |> serialize_raw(t.planes_trimmed) arch |> serialize_raw(t.experts_mx4) arch |> serialize_raw(t.ple_model_bf16) arch |> serialize_raw(t.kquant_native) @@ -293,9 +492,9 @@ def private serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.mtp_headnorm_off) } -// the tripwire: serialize_image_meta(Model) covers 64 fields; blocks + image_map + +// the tripwire: serialize_image_meta(Model) covers 66 fields; blocks + image_map + // image_bytes are deliberate skips. A new non-array Model field trips this until BOTH agree. -let IMAGE_META_FIELDS = 65 + 3 +let IMAGE_META_FIELDS = 66 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -321,61 +520,69 @@ def private image_post_load(var t : Model) { // ===== save ===== -// one plane: pad to a page, bulk-write, account. long_ forms: a plane past 2 GiB -// (voxtral-mini qblob = 3.6 G elements) does not fit the int32 length/byte-count rail -def private write_plane(f : file; var sections : array; var cur : uint64&; - var ok : bool&; name : string; var field : array) { - if (empty(field) || !ok) { // a dead save stays dead — don't keep pushing bytes at a full disk +// one plane: pad to a page, bulk-append, account. The writer advances w.cur by exactly what it +// accepted, so the section table cannot drift from the file. long_ forms: a plane past 2 GiB +// (voxtral-mini qblob = 3.6 G elements) does not fit the int32 rail. +def private write_plane(var w : ImgWriter; var sections : array; + name : string; var field : array) { + if (empty(field) || !w.ok) { return } let nbytes = uint64(long_length(field)) * uint64(typeinfo sizeof(field[0])) - let at = pad_to_page(cur) - write_zeros(f, at - cur) - let wrote = f |> long_fwrite(field) - if (uint64(wrote) != nbytes) { - // a short write is an I/O failure (ENOSPC in practice), not an accounting bug - to_log(LOG_ERROR, "dasLLAMA image: plane '{name}' short write {wrote} of {int64(nbytes)} bytes at offset {int64(at)} — disk full? image save aborted\n") - ok = false + let at = pad_to_page(w.cur) + w_zeros(w, at - w.cur) + let ts = ref_time_ticks() + unsafe { + w_append(w, addr(field[0]), nbytes) + } + // big planes get their own line: a uniform rate means the cost is systemic, a few slow ones + // mean stalls. Only >=256 MB, so small models stay quiet and this needs no env knob. + if (nbytes >= 256ul * 1024ul * 1024ul) { + let ms = int64(get_time_usec(ts)) / 1000l + let mb = int64(nbytes) >> 20l + to_log(LOG_INFO, "dasLLAMA image: plane {name} {mb} MB in {ms} ms = {ms > 0l ? mb * 1000l / ms : 0l} MB/s\n") + } + if (!w.ok) { + to_log(LOG_ERROR, "dasLLAMA image: plane '{name}' failed writing {int64(nbytes)} bytes at offset {int64(at)} — disk full? image save aborted\n") return } sections |> push(ImgSection(name = name, off = at, bytes = nbytes)) - cur = at + nbytes - // fwrite and the section table must agree byte-for-byte, or the mapped planes land at - // the wrong offsets — fail the save loudly, not the load - if (uint64(ftell(f)) != cur) { - to_log(LOG_ERROR, "dasLLAMA image: plane '{name}' wrote {ftell(f)} != accounted {int64(cur)} (nbytes {int64(nbytes)}) — image save aborted\n") - ok = false - } } //! Serialize a loaded weight-carrying struct as a prepared image: array planes page-aligned raw //! (borrowed back at load), everything else archived. Tmp+rename: atomic on POSIX; on Windows a //! brief missing-file window lets a concurrent reader regenerate. Caller supplies `serialize_image_meta`/`image_post_load` (`_::` dispatch). -def save_image(var t; path : string; tag : string = "") : bool { +def save_image(var t; path : string; tag : string = ""; quant : string = "q8") : bool { // per-writer tmp name: concurrent savers (parallel test workers on one cache miss) must // not clobber each other's half-written file; last atomic rename wins with same content let tmp_path = "{path}.{ref_time_ticks()}.tmp" - var ok = true + // one config snapshot serves the meta embed AND the header hash — they must agree + let dcfg = dlim_config_current(quant) + let ident = dlim_identity(dcfg, IMAGE_VERSION, tag) var fend = 0ul + var meta_off = 0ul + var meta_bytes = 0ul + var nsections = 0 var meta <- new MemSerializer() var warch = Archive(reading = false, stream = meta) var sections : array - fopen(tmp_path, "wb") $(f) { - if (f == null) { - ok = false - return - } - // page 0: header placeholder — patched after the walk (fseek back) - write_zeros(f, uint64(IMAGE_PAGE)) - var cur = uint64(IMAGE_PAGE) - let ts_meta = ref_time_ticks() - _::serialize_image_meta(warch, t) - to_log(LOG_INFO, "dasLLAMA image: meta serialize {get_time_usec(ts_meta) / 1000}ms\n") + // meta first: it needs nothing from the planes, and having it sized up front is what lets + // the writer preallocate the exact file (see image_total_bytes) + let ts_meta = ref_time_ticks() + _::serialize_image_meta(warch, t) + to_log(LOG_INFO, "dasLLAMA image: meta serialize {get_time_usec(ts_meta) / 1000}ms\n") + var scalar_blob <- meta->extractData() + unsafe { + delete meta + } + var w = w_open(tmp_path, image_total_bytes(t, uint64(long_length(scalar_blob)) + SECTION_TABLE_SLACK)) + if (w.ok) { + w_zeros(w, uint64(IMAGE_PAGE)) // page 0: header placeholder — patched after close let ts_planes = ref_time_ticks() apply(t) $(name : string; var field) { static_if (typeinfo is_array(field)) { static_if (!typeinfo is_string(field[0])) { // string arrays ride the meta blob - write_plane(f, sections, cur, ok, name, field) + write_plane(w, sections, name, field) } } static_elif (typeinfo safe_has_field < image_map > (field)) { // a nested weight-carrier on the image rail (marker: it declares image_map) — @@ -384,54 +591,82 @@ def save_image(var t; path : string; tag : string = "") : bool { apply(field) $(iname : string; var ifield) { static_if (typeinfo is_array(ifield)) { static_if (!typeinfo is_string(ifield[0])) { - write_plane(f, sections, cur, ok, "{name}.{iname}", ifield) + write_plane(w, sections, "{name}.{iname}", ifield) } } } } } - to_log(LOG_INFO, "dasLLAMA image: planes write {get_time_usec(ts_planes) / 1000}ms\n") - // meta = [sections][the walk's scalar stream], one blob at the tail + let planes_ms = int64(get_time_usec(ts_planes)) / 1000l + let planes_mb = int64(w.cur) >> 20l + to_log(LOG_INFO, "dasLLAMA image: planes write {planes_ms}ms ({planes_mb} MB at {planes_ms > 0l ? planes_mb * 1000l / planes_ms : 0l} MB/s)\n") + // the writer's own split — staging our bytes vs waiting on the device. Anything the two + // do not account for is time spent OUTSIDE the writer (the walk itself). + let copy_ms = int64(unsafe(dwrite_stat(w.h, 0))) / 1000000l + let write_ms = int64(unsafe(dwrite_stat(w.h, 1))) / 1000000l + let direct_mb = int64(unsafe(dwrite_stat(w.h, 2))) >> 20l + let bounce_mb = int64(unsafe(dwrite_stat(w.h, 3))) >> 20l + to_log(LOG_INFO, "dasLLAMA image: writer split: stage {copy_ms}ms ({bounce_mb} MB bounced), syscall {write_ms}ms ({direct_mb} MB direct), unaccounted {planes_ms - copy_ms - write_ms}ms\n") + // meta = [identity string][config JSON][sections][the walk's scalar stream], one blob at + // the tail — the two strings lead so a mismatched load can print WHAT the image was + // baked for without parsing anything else var smeta <- new MemSerializer() var sarch = Archive(reading = false, stream = smeta) + var m_ident = ident + var m_cfg = dlim_config_json(dcfg) + sarch |> serialize(m_ident) + sarch |> serialize(m_cfg) sarch |> serialize(sections) var meta_blob <- smeta->extractData() - var scalar_blob <- meta->extractData() meta_blob |> push_from(scalar_blob) - let meta_off = cur - let meta_bytes = uint64(long_length(meta_blob)) - f |> fwrite(meta_blob) - cur += meta_bytes + nsections = length(sections) + meta_off = w.cur + meta_bytes = uint64(long_length(meta_blob)) + if (meta_bytes > 0ul) { // never empty in practice — the section table always rides here + unsafe { + w_append(w, addr(meta_blob[0]), meta_bytes) + } + } // pad the file itself to a page multiple (the whole-file bytesNoCopy wrap needs it) - fend = pad_to_page(cur) - write_zeros(f, fend - cur) - // patch the header + fend = pad_to_page(w.cur) + w_zeros(w, fend - w.cur) + delete meta_blob + unsafe { + delete smeta + } + delete sections + } + delete scalar_blob + var ok = w_close(w) + if (ok) { + // the header carries counts only the finished walk knows (section count, meta offset and + // size), and the direct writer is append-only — so patch page 0 through an ordinary + // handle now that the streaming one is closed var hdr : array hdr |> resize(int(IMAGE_HEADER_BYTES)) store_u32(hdr, 0, IMAGE_MAGIC) store_u32(hdr, 4, uint(IMAGE_VERSION)) store_u32(hdr, 8, uint(IMAGE_PAGE)) - store_u32(hdr, 12, uint(length(sections))) + store_u32(hdr, 12, uint(nsections)) store_u64(hdr, 16, meta_off) store_u64(hdr, 24, meta_bytes) store_u64(hdr, 32, fend) - store_u64(hdr, 40, hash(image_identity(tag))) - fseek(f, 0l, seek_set) - f |> fwrite(hdr) - delete hdr - delete meta_blob - delete scalar_blob - unsafe { - delete smeta + store_u64(hdr, 40, hash(ident)) + ok = false + fopen(tmp_path, "r+b") $(hf) { + if (hf != null) { + fseek(hf, 0l, seek_set) + ok = int64(hf |> fwrite(hdr)) == IMAGE_HEADER_BYTES + } } - delete sections - } - unsafe { - delete meta + if (!ok) { + to_log(LOG_ERROR, "dasLLAMA image: '{tmp_path}' header patch failed — image save aborted\n") + } + delete hdr } if (ok) { - // buffered-tail backstop: the small tail writes can sit in the stdio buffer and hit - // ENOSPC only at fclose, which reports nothing here — verify what landed on disk + // last backstop: the writer reports its own failures, but an ENOSPC that only surfaces + // at close would still leave a short file — verify what actually landed on disk var st : FStat if (!stat(tmp_path, st) || st.size != fend) { to_log(LOG_ERROR, "dasLLAMA image: '{tmp_path}' is {int64(st.size)} bytes on disk, expected {int64(fend)} — disk full? image save aborted\n") @@ -439,7 +674,8 @@ def save_image(var t; path : string; tag : string = "") : bool { } } if (!ok) { - if (!remove(tmp_path)) { + // a failed OPEN never created the file, so only warn about a tmp that is actually there + if (stat(tmp_path).is_valid && !remove(tmp_path)) { to_log(LOG_WARNING, "dasLLAMA image: orphaned tmp '{tmp_path}' could not be removed — clean it up by hand\n") } return false @@ -461,8 +697,143 @@ def save_image(var t; path : string; tag : string = "") : bool { //! Model entry point (the AudioTower one is load_audio_tower's cache in dasllama_audio). //! `tag` names the flavor (METAL_IMAGE_TAG for blob-only metal-flavor models, "" planar). -def save_model_image(var t : Model; path : string; tag : string = "") : bool { - return save_image(t, path, tag) +def save_model_image(var t : Model; path : string; tag : string = ""; quant : string = "q8") : bool { + return save_image(t, path, tag, quant) +} + +//! save_image's STREAMING twin (Model-only): fields with recorded FillJobs stream through +//! fill_stream_plane; empty `jobs` degrades to the eager save exactly. Call from +//! load_gguf_streaming's block (m/bytes must be alive); keep the tails in step with save_image. +def save_model_image_streaming(var t : Model; m : GGUFMeta; bytes : array | #; + var jobs : array; path : string; tag : string = ""; quant : string = "q8") : bool { + let tmp_path = "{path}.{ref_time_ticks()}.tmp" + let dcfg = dlim_config_current(quant) + let ident = dlim_identity(dcfg, IMAGE_VERSION, tag) + var fend = 0ul + var meta_off = 0ul + var meta_bytes = 0ul + var nsections = 0 + var meta <- new MemSerializer() + var warch = Archive(reading = false, stream = meta) + var sections : array + let ts_meta = ref_time_ticks() + serialize_image_meta(warch, t) + to_log(LOG_INFO, "dasLLAMA image: meta serialize {get_time_usec(ts_meta) / 1000}ms\n") + var scalar_blob <- meta->extractData() + unsafe { + delete meta + } + // preallocation: the generic total (streamed fields are empty, contributing 0) plus each + // streamed plane's exact bytes + a page of padding, plus the pending-wscale qscales16 + // (absent until the epilogue). Over-reserve is free — dwrite_close truncates. + var extra = 0ul + for (fname in ["qblob", "k4q", "k5q", "k6q", "q40q", "q51q"]) { + let sb = stream_plane_bytes(t, jobs, fname) + if (sb > 0l) { + extra += uint64(sb) + uint64(IMAGE_PAGE) + } + } + if (t.wscale_f16 && empty(t.qscales16) && !empty(t.qscales)) { + extra += uint64(long_length(t.qscales) * 2l) + uint64(IMAGE_PAGE) + } + var regs : array + stream_collect_regions(t, regs) + var fill_ok = true + var w = w_open(tmp_path, image_total_bytes(t, uint64(long_length(scalar_blob)) + SECTION_TABLE_SLACK) + extra) + if (w.ok) { + w_zeros(w, uint64(IMAGE_PAGE)) // page 0: header placeholder — patched after close + let ts_planes = ref_time_ticks() + apply(t) $(name : string; var field) { + static_if (typeinfo is_array(field)) { + static_if (!typeinfo is_string(field[0])) { + if (stream_plane_bytes(t, jobs, name) > 0l) { + let padded = pad_to_page(w.cur) + w_zeros(w, padded - w.cur) + let at = w.cur + if (!fill_stream_plane(m, bytes, t, name, jobs, regs) $(p : void?; nb : uint64) { w_append(w, p, nb) }) { + fill_ok = false + } + sections |> push(ImgSection(name = name, off = at, bytes = w.cur - at)) + } else { + write_plane(w, sections, name, field) + } + } + } + } + let planes_ms = int64(get_time_usec(ts_planes)) / 1000l + let planes_mb = int64(w.cur) >> 20l + to_log(LOG_INFO, "dasLLAMA image: streamed planes write {planes_ms}ms ({planes_mb} MB at {planes_ms > 0l ? planes_mb * 1000l / planes_ms : 0l} MB/s)\n") + var smeta <- new MemSerializer() + var sarch = Archive(reading = false, stream = smeta) + var m_ident = ident + var m_cfg = dlim_config_json(dcfg) + sarch |> serialize(m_ident) + sarch |> serialize(m_cfg) + sarch |> serialize(sections) + var meta_blob <- smeta->extractData() + meta_blob |> push_from(scalar_blob) + nsections = length(sections) + meta_off = w.cur + meta_bytes = uint64(long_length(meta_blob)) + if (meta_bytes > 0ul) { + unsafe { + w_append(w, addr(meta_blob[0]), meta_bytes) + } + } + fend = pad_to_page(w.cur) + w_zeros(w, fend - w.cur) + delete meta_blob + unsafe { + delete smeta + } + delete sections + } + delete scalar_blob + delete regs + var ok = w_close(w) && fill_ok + if (ok) { + var hdr : array + hdr |> resize(int(IMAGE_HEADER_BYTES)) + store_u32(hdr, 0, IMAGE_MAGIC) + store_u32(hdr, 4, uint(IMAGE_VERSION)) + store_u32(hdr, 8, uint(IMAGE_PAGE)) + store_u32(hdr, 12, uint(nsections)) + store_u64(hdr, 16, meta_off) + store_u64(hdr, 24, meta_bytes) + store_u64(hdr, 32, fend) + store_u64(hdr, 40, hash(ident)) + ok = false + fopen(tmp_path, "r+b") $(hf) { + if (hf != null) { + fseek(hf, 0l, seek_set) + ok = int64(hf |> fwrite(hdr)) == IMAGE_HEADER_BYTES + } + } + if (!ok) { + to_log(LOG_ERROR, "dasLLAMA image: '{tmp_path}' header patch failed — image save aborted\n") + } + delete hdr + } + if (ok) { + var st : FStat + if (!stat(tmp_path, st) || st.size != fend) { + to_log(LOG_ERROR, "dasLLAMA image: '{tmp_path}' is {int64(st.size)} bytes on disk, expected {int64(fend)} — disk full? image save aborted\n") + ok = false + } + } + if (!ok) { + if (stat(tmp_path).is_valid && !remove(tmp_path)) { + to_log(LOG_WARNING, "dasLLAMA image: orphaned tmp '{tmp_path}' could not be removed — clean it up by hand\n") + } + return false + } + // POSIX rename-over is atomic; Windows needs remove-then-rename (a concurrent reader may + // regenerate in the window) + if (rename(tmp_path, path) || (remove(path) && rename(tmp_path, path))) { + return true + } + to_log(LOG_ERROR, "dasLLAMA image: could not move '{tmp_path}' over '{path}'\n") + return false } // ===== load ===== @@ -490,7 +861,7 @@ def private borrow_plane(bp : uint8 const?; msize : uint64; path : string; //! Map a prepared image and rebuild the struct with zero O(model) work: array plane fields borrow //! the mapping (finalizer forgets + unmaps), archive blob restores everything else. false = absent / //! wrong identity / wrong version — caller regenerates from gguf. Same `_::` contract as save_image. -def load_image(path : string; var t; tag : string = "") : bool { +def load_image(path : string; var t; tag : string = ""; quant : string = "q8") : bool { if (!stat(path).is_valid) { return false } @@ -512,11 +883,12 @@ def load_image(path : string; var t; tag : string = "") : bool { return false } var bp = unsafe(reinterpret(base)) - if ((read_u32(bp, 0l) != IMAGE_MAGIC || read_u32(bp, 4l) != uint(IMAGE_VERSION)) || - (read_u32(bp, 8l) != uint(IMAGE_PAGE) || read_u64(bp, 40l) != hash(image_identity(tag)))) { - // every decline is loud (short of the file simply not existing) — a silently - // regenerating cache is undebuggable - to_log(LOG_WARNING, "dasLLAMA image: '{path}' declined — size {msize} magic {read_u32(bp, 0l)} version {read_u32(bp, 4l)} (want {IMAGE_VERSION}) page {read_u32(bp, 8l)} identity {read_u64(bp, 40l)} (want {hash(image_identity(tag))} = '{image_identity(tag)}')\n") + if (read_u32(bp, 0l) != IMAGE_MAGIC || read_u32(bp, 4l) != uint(IMAGE_VERSION) + || read_u32(bp, 8l) != uint(IMAGE_PAGE)) { + // every decline is loud (short of a missing file) — a silently regenerating cache is + // undebuggable. A wrong version means the meta layout is unknowable, so this arm + // cannot read the baked identity string the identity-mismatch arm below prints. + to_log(LOG_WARNING, "dasLLAMA image: '{path}' declined — size {msize} magic {read_u32(bp, 0l)} version {read_u32(bp, 4l)} (want {IMAGE_VERSION}) page {read_u32(bp, 8l)}\n") unsafe(fmap_close(base, msize)) return false } @@ -532,6 +904,27 @@ def load_image(path : string; var t; tag : string = "") : bool { unsafe(fmap_close(base, msize)) return false } + let want_ident = image_identity(tag, quant) + if (read_u64(bp, 40l) != hash(want_ident)) { + // same version ⇒ the meta head is readable: say WHAT the image was baked for, not + // just that a 64-bit hash missed (bounds were validated above) + var head : array + head |> resize(int(min(meta_bytes, 65536ul))) + unsafe { + memcpy(addr(head[0]), reinterpret(bp + int64(meta_off)), uint64(long_length(head))) + } + var hmeta <- new MemSerializer(head) + var harch = Archive(reading = true, stream = hmeta) + var baked_ident = "" + harch |> serialize(baked_ident) + to_log(LOG_WARNING, "dasLLAMA image: '{path}' declined — baked for '{hmeta->OK() ? baked_ident : "(unreadable)"}', this run is '{want_ident}'\n") + delete head + unsafe { + delete hmeta + } + unsafe(fmap_close(base, msize)) + return false + } // the meta blob is the ONE copied+parsed piece (small); planes stay lazy in the mapping var meta_blob : array meta_blob |> resize(int(meta_bytes)) @@ -540,6 +933,10 @@ def load_image(path : string; var t; tag : string = "") : bool { } var meta <- new MemSerializer(meta_blob) var rarch = Archive(reading = true, stream = meta) + var baked_ident = "" + var baked_cfg = "" + rarch |> serialize(baked_ident) + rarch |> serialize(baked_cfg) var sections : array rarch |> serialize(sections) // the header count must agree with the meta blob's own section list — a disagreement @@ -645,8 +1042,8 @@ def private guard_interp_gguf_load(path : string) { } //! Model entry point — see load_image (`tag` = the flavor, like save_model_image). -def load_model_image(path : string; var t : Model; tag : string = "") : bool { - if (!load_image(path, t, tag)) { +def load_model_image(path : string; var t : Model; tag : string = ""; quant : string = "q8") : bool { + if (!load_image(path, t, tag, quant)) { return false } // AFTER the plane borrows (image_post_load runs before them, when every array is still @@ -668,7 +1065,7 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32) : Model var t = Model() let metal_first = get_metal_mode() != MetalMode.off && has_decode_override("metal") let vk_cfg = metal_first ? "" : moe_gpu_bake_tag() - let vk_tag = vk_cfg == "" ? "" : "vulkan|{vk_cfg}" + let vk_tag = vk_cfg // dlim_vulkan_tag already carries the "vulkan|" prefix ("" = tier absent) var loaded = false if (vk_tag != "") { vulkan_bake_slice_begin() @@ -708,7 +1105,7 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32) : Model // the vulkan flavor: the GPU walk slices the baked device-layout planes off the mapping // instead of re-gathering; ANY vulkan config switch changes the tag and rebakes below let vk_cfg = want_metal ? "" : moe_gpu_bake_tag() - let vk_tag = vk_cfg == "" ? "" : "vulkan|{vk_cfg}" + let vk_tag = vk_cfg // dlim_vulkan_tag already carries the "vulkan|" prefix ("" = tier absent) if (vk_tag != "") { let img_v = image_path_for(path, vk_tag) vulkan_bake_slice_begin() diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index e968f01a5a..40a3db13e6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -196,35 +196,28 @@ def private compact_kq_scale_strip(var ks : array) { // ===== CPU repack walkers (backend grp interleave, in place at load) ===== -// A repack work item: one mr-aligned row-chunk of a weight region. fmt: 0 = q8 (qblob/qscales), -// 1 = mx4 (mxq/mxs), 4/5/6 = the kq plane pairs. Chunking splits huge regions (the classifier) for -// load balance; 1024 is a multiple of every backend mr, so row groups never straddle a boundary. -struct private RepackReg { - off : int64 - n : int64 - d : int64 - fmt : int -} - -def private push_repack(var regs : array; fmt : int; off, n, d : int64) { +// A repack work item is common's StreamReg (fmt: 0 = q8, 1 = mx4, 2 = q51, 4/5/6/40 = kq). +// Chunking splits huge regions (the classifier) for load balance; 1024 is a multiple of every +// backend mr, so row groups never straddle a boundary. +def private push_repack(var regs : array; fmt : int; off, n, d : int64) { var r0 = 0l while (r0 < d) { let rc = min(1024l, d - r0) - regs |> push(RepackReg(off = off + r0 * n, n = n, d = rc, fmt = fmt)) + regs |> push(StreamReg(off = off + r0 * n, n = n, d = rc, fmt = fmt)) r0 += rc } } // Run every collected repack across the jobque — regions are disjoint, so any lane count is // bit-exact. Workers touch hoisted raw pointers + the hoisted backend fn values only. -def private repack_regions(var t : Model; regs : array) { +def private repack_regions(var t : Model; regs : array) { if (empty(regs)) { return } let rq8 = g_repack_q8q8 let rkq = g_repack_kq let rmx = g_repack_mx4 - let rq51 = g_repack_q51 + let rq51 = active_repack_q51() // override-aware: a cross-box q51 target needs the gen generic unsafe { var qbp : int8? = null var qsp : float? = null @@ -254,7 +247,7 @@ def private repack_regions(var t : Model; regs : array) { if (!empty(t.q40s)) { q40sp = addr(t.q40s[0]) } if (!empty(t.q51q)) { q51qp = addr(t.q51q[0]) } if (!empty(t.q51s)) { q51sp = addr(t.q51s[0]) } - let rp = addr(regs[0]) + let rp = addr(regs[0]) let nr = length(regs) let njobs = is_job_que_available() ? min(nr, 4 * (get_total_hw_jobs() + 1)) : 1 maybe_parallel_for(0, nr, njobs) $(rb, re) { @@ -284,11 +277,10 @@ def private repack_regions(var t : Model; regs : array) { // Repack every Q8 2D weight tensor in qblob/qscales into the active backend's interleaved layout, // IN PLACE, once at load. Mirrors forward()'s per-tensor offsets/shapes exactly; the embedding + // norms live in fblob and are never touched. -def private repack_q8_weights(var t : Model) { +def private collect_q8_regions(t : Model; var regs : array) { let c = t.config let dim = c.dim let hidden = c.hidden_dim - var regs : array for (l in range64(c.n_layers + c.n_layer_nextn)) { // + the MTP/NextN block at index n_layers let kv = layer_kv_dim(c, l) let qd = layer_qd(c, l) @@ -379,13 +371,18 @@ def private repack_q8_weights(var t : Model) { if (c.n_layer_nextn > 0l && t.mtp_ehproj_fmt == KqFmt.q8) { // NextN eh_proj: 2*dim -> dim push_repack(regs, 0, t.mtp_ehproj_off, 2l * dim, dim) } +} + +def private repack_q8_weights(var t : Model) { + var regs : array + collect_q8_regions(t, regs) repack_regions(t, regs) t.mx4_repacked = true delete regs } // route one kq-tagged tensor to its format's plane pair (q8 tags no-op — those live in qblob) -def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d : int64) { +def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d : int64) { if (fmt == KqFmt.k4) { push_repack(regs, 4, woff, n, d) } elif (fmt == KqFmt.k5) { @@ -400,10 +397,9 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d // Repack every K-quant-tagged weight tensor into the active backend's grp kq layout, IN // PLACE — the kq sibling of repack_q8_weights, called only when the backend carries kq slots. // The tied/untied classifier repacks too: embed_row gathers its rows through the grp layout after. -def private repack_kq_weights(var t : Model) { +def private collect_kq_regions(t : Model; var regs : array) { let c = t.config let dim = c.dim - var regs : array for (l in range64(c.n_layers + c.n_layer_nextn)) { // + the MTP/NextN block at index n_layers let kv = layer_kv_dim(c, l) let qd = layer_qd(c, l) @@ -454,6 +450,11 @@ def private repack_kq_weights(var t : Model) { if (c.n_layer_nextn > 0l) { // NextN eh_proj (no-op for q8 tags) push_repack_kq(regs, t.mtp_ehproj_fmt, t.mtp_ehproj_off, 2l * dim, dim) } +} + +def private repack_kq_weights(var t : Model) { + var regs : array + collect_kq_regions(t, regs) repack_regions(t, regs) delete regs } @@ -465,7 +466,7 @@ def private repack_mx4_stacks(var t : Model) { let c = t.config let dim = c.dim let ege = dim * c.n_ff_exp - var regs : array + var regs : array for (l in range64(c.n_layers)) { for (e in range64(c.n_expert)) { let base = e * ege @@ -480,11 +481,10 @@ def private repack_mx4_stacks(var t : Model) { // q51-native expert stacks -> the active backend's grp q51 layout (per-stack: a model can // mix q51 downs with k4 gates, e.g. gemma-4-26B-A4B). No-op unless a stack is tagged q51. -def private repack_q51_stacks(var t : Model) { +def private collect_q51_regions(t : Model; var regs : array) { let c = t.config let dim = c.dim let ege = dim * c.n_ff_exp - var regs : array for (l in range64(c.n_layers)) { let f1q = fmt_at(t.we1_fmt, l) == KqFmt.q51 let f2q = fmt_at(t.we2_fmt, l) == KqFmt.q51 @@ -505,6 +505,11 @@ def private repack_q51_stacks(var t : Model) { } } } +} + +def private repack_q51_stacks(var t : Model) { + var regs : array + collect_q51_regions(t, regs) if (!empty(regs)) { repack_regions(t, regs) to_log(LOG_INFO, "dasLLAMA: q51 expert planes repacked to grp{active_q51_layout_mr()}\n") @@ -512,6 +517,20 @@ def private repack_q51_stacks(var t : Model) { delete regs } +// StreamRepackFn: one region through the active backend's repack, on caller-supplied pointers +// (the streamed fill biases the q pointer into its temp; scales stay in the RAM plane) +def private stream_repack_one(fmt : int; qp, sp : void?; n, d : int64) { + unsafe { + if (fmt == 0) { + invoke(g_repack_q8q8, reinterpret(qp), reinterpret(sp), n, d) + } elif (fmt == 2) { + invoke(active_repack_q51(), reinterpret(qp), reinterpret(sp), n, d) + } else { + invoke(g_repack_kq, fmt, reinterpret(qp), reinterpret(sp), n, d) + } + } +} + // ===== GPU tier gathers (prepared planes -> device-layout bytes) ===== // Gather one PREPARED Q8 weight stack into the GPU tier's two row-major planes — int8 quants + @@ -817,4 +836,5 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice def private register_layout_hooks { register_model_layout(@@repack_q8_weights, @@repack_kq_weights, @@repack_mx4_stacks, @@repack_q51_stacks, @@moe_gpu_gather_stack, @@moe_gpu_gather_stack_kq, @@moe_gpu_gather_stack_q8n) + register_stream_layout(@@collect_q8_regions, @@collect_kq_regions, @@collect_q51_regions, @@stream_repack_one) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 98287e1a8c..3b87162414 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -6,6 +6,7 @@ module dasllama_math shared public require math require daslib/fio // has_env_variable: DAS_JOBQUE_TEAM_RANK_GATE suppresses the profile's team_rank_gate knob require dasllama/dasllama_env // env_flag/env_int64: the shared typed readers + the knob registry +require dasllama/dasllama_config // DlimVulkanConfig — the dry-bake seam's config argument require daslib/jobque_boost public // matmul's threaded path expands parallel_for; consumers need its symbols require daslib/jobque_profile // named trace categories + unit markers (set_trace_tags registers them) require dasllama/dasllama_par // maybe_parallel_for: thread-or-inline a kernel body on a runtime size flag @@ -2119,6 +2120,12 @@ def public gpu_want_qkv : bool def public gpu_want_cls : bool => env_flag("DASLLAMA_GPU_CLS", false) +//! The classifier UPLOAD gate (opt-OUT, default on when the tier is armed) — distinct from +//! [[gpu_want_cls]], the opt-IN arm signal. One definition, shared by the upload walk and the +//! DlimConfiguration identity, so the two can never disagree on the default again. +def public gpu_want_cls_upload : bool + => env_flag("DASLLAMA_GPU_CLS", true) + //! A GPU backend registers its arm probe here at ``[init]`` so config-driven arming works //! after module init (the env path arms at ``[init]`` directly). def public set_moe_gpu_arm(name : string; f : function<() : bool>) { @@ -2163,6 +2170,71 @@ def public set_moe_gpu_expert_q8n_probe(f : function<() : bool>) { g_moe_gpu_expert_q8n = f } +[unused_argument(v)] +def private moe_gpu_no_dry_bake(v : DlimVulkanConfig) : bool => false + +var private g_moe_gpu_dry_bake = @@moe_gpu_no_dry_bake + +//! A GPU tier registers its OFFLINE-bake arm here: the load walk runs its full accept/refuse +//! arithmetic against the config's numbers, gathers collect, and no device call is made. +def public set_moe_gpu_dry_bake_arm(f : function<(v : DlimVulkanConfig) : bool>) { + g_moe_gpu_dry_bake = f +} + +//! Arm the offline bake (false = no GPU backend in this build, or the config disagrees with +//! this process' env wants — the tier logs the exact mismatch). +def public moe_gpu_dry_bake_arm(v : DlimVulkanConfig) : bool => invoke(g_moe_gpu_dry_bake, v) + +//! A baked-plan gather's mapped-file window (P2 imported mirrors): pointers into the flavor +//! image's mapping + plane byte sizes. wq == null = no window recorded for (woff, fmt). +typedef MoeGpuSliceWindow = tuple + +[unused_argument(woff, fmt)] +def private moe_gpu_no_slice_window(woff : int64; fmt : int) : MoeGpuSliceWindow { + return default +} + +var private g_moe_gpu_slice_window = @@moe_gpu_no_slice_window + +//! The loader registers its slice-window provider (populated while slicing a flavor image); +//! a GPU tier asks through [[moe_gpu_slice_window]] to import instead of copying. +def public set_moe_gpu_slice_window_fn(f : function<(woff : int64; fmt : int) : MoeGpuSliceWindow>) { + g_moe_gpu_slice_window = f +} + +def public moe_gpu_slice_window(woff : int64; fmt : int) : MoeGpuSliceWindow => invoke(g_moe_gpu_slice_window, woff, fmt) + +[unused_argument(base, bytes)] +def private moe_gpu_no_unmap_notify(base : void?; bytes : uint64) {} + +var private g_moe_gpu_unmap_notify = @@moe_gpu_no_unmap_notify + +//! A GPU tier registers this to learn a model's mapping is about to unmap — imported device +//! memory over that mapping must be released FIRST (the Model finalizer invokes it). +def public set_moe_gpu_unmap_notify(f : function<(base : void?; bytes : uint64) : void>) { + g_moe_gpu_unmap_notify = f +} + +def public moe_gpu_unmap_notify(base : void?; bytes : uint64) { + invoke(g_moe_gpu_unmap_notify, base, bytes) +} + +[unused_argument(n, rows, fmt, stream)] +def private moe_gpu_always_accept(n : int64; rows : int64; fmt : int; stream : bool) : bool => true + +var private g_moe_gpu_would_accept = @@moe_gpu_always_accept + +//! A GPU tier registers its accept arithmetic (no side effects). The slice replay asks through +//! [[moe_gpu_would_accept]] whether a gather the plan lacks is a refusal boundary — the bake +//! never records refused gathers, so the walk's boundary attempt must re-refuse, not diverge. +def public set_moe_gpu_would_accept_fn(f : function<(n : int64; rows : int64; fmt : int; stream : bool) : bool>) { + g_moe_gpu_would_accept = f +} + +def public moe_gpu_would_accept(n : int64; rows : int64; fmt : int; stream : bool) : bool { + return invoke(g_moe_gpu_would_accept, n, rows, fmt, stream) +} + def private moe_gpu_no_bake_tag : string => "" var private g_moe_gpu_bake_tag = @@moe_gpu_no_bake_tag @@ -2250,13 +2322,17 @@ def public moe_gpu_upload_stack(wq : array | #; ws : array | #; wo if (!g_moe_gpu_installed) { return false } - var wsp : uint8 const? // q8n stacks carry no scale plane — an empty ws hands the tier null + // empty planes hand the tier null: q8n stacks carry no scale plane, and the slice replay's + // refusal-boundary serve is empty on purpose (the tier refuses on arithmetic, no bytes read) + var wqp : uint8 const? + if (!empty(wq)) { + wqp = unsafe(addr(wq[0])) + } + var wsp : uint8 const? if (!empty(ws)) { wsp = unsafe(addr(ws[0])) } - unsafe { - return invoke(g_moe_gpu_upload, addr(wq[0]), wsp, woff, n, rows, fmt, stream) - } + return invoke(g_moe_gpu_upload, wqp, wsp, woff, n, rows, fmt, stream) } //! Fused MoE FFN on the GPU tier — the whole gate+up -> act -> requant -> down chain of the @@ -2623,27 +2699,77 @@ def private kq_layout_grp4_fmt(fmt : int) : int => 4 def private q8_wbias_zero() : int => 0 def private q8_kgroup_four() : int => 4 +// ===== cross-box bake: repack by NUMBERS ===== +// The getters below are the one layout authority; a foreign-backend bake overrides them with the target's numbers — the generic grp repacks produce the exact bytes, and the local kernels never serve them (the foreign identity cannot load here) +var private g_bake_cpu_override = false +var private g_bake_cpu = DlimCpuConfig() + +//! The backend name the dlim identity reports — the bake alias when a cross-box bake is armed. +def dlim_backend_name() : string => g_bake_cpu_override ? g_bake_cpu.backend : active_kernel_backend() + +//! Pin the backend a bake targets: a registered name pins normally; a FOREIGN name arms +//! repack-by-numbers (the gen tier's generic grp primitives produce the target layout, +//! the identity reports the target backend). false = no route — the caller fail-closes. +def pin_backend_for_bake(cpu : DlimCpuConfig) : bool { + if (key_exists(g_kernel_backends, cpu.backend)) { + pin_kernel_backend(cpu.backend) + return active_kernel_backend() == cpu.backend + } + select_matmul_backend_for_load() // the load enters repack stages only under a repack backend + if (!kernel_backend_needs_repack(active_kernel_backend())) { + to_log(LOG_WARNING, "dasLLAMA: cross-box bake for '{cpu.backend}' needs the generated repack family — none is active here\n") + return false + } + g_bake_cpu = cpu + g_bake_cpu_override = true + return true +} + +def clear_bake_backend_override() { + g_bake_cpu_override = false +} + //! The Q8 interleave (mr) of the ACTIVE backend's weight layout — cached at activation from //! KernelBackend.q8_layout (4 for every hand tier; the stamped mr on the gen backend). -def active_q8_layout_mr() : int64 => g_active_q8_layout_mr +def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_active_q8_layout_mr //! The active backend's kq plane interleave for `fmt` (4/5/6) — per-format since the kq v2 //! family split; the loader freezes these onto the Model at repack time. -def active_kq_layout_mr(fmt : int) : int64 => int64(invoke(g_active_kq_layout, fmt)) +def active_kq_layout_mr(fmt : int) : int64 { + if (g_bake_cpu_override) { + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : g_bake_cpu.kq_mr6)) + } + return int64(invoke(g_active_kq_layout, fmt)) +} //! The active backend's q51 plane interleave — 1 (disk order) on every hand tier; the stamped //! mr on the arm64-gen backend (the self-anchored q51 GEMV family). -def active_q51_layout_mr() : int64 => g_active_q51_layout_mr +def active_q51_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q51_mr : g_active_q51_layout_mr //! The weight-plane bias of the ACTIVE backend's Q8 layout — cached at activation from //! KernelBackend.q8_wbias (0 for every hand tier; 128 on a bias128 gen stamp). The mx4->Q8 //! expand biases its LUT writes to match. -def active_q8_wbias() : int64 => g_active_q8_wbias +def active_q8_wbias() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_wbias : g_active_q8_wbias //! The plane kgroup of the ACTIVE backend's Q8 layout — cached at activation from //! KernelBackend.q8_kgroup (4 for every hand tier; 8 on an smmla gen stamp). The mx4->Q8 //! expand's positional map and the batch token-tail generic read it. -def active_q8_kgroup() : int64 => g_active_q8_kgroup +def active_q8_kgroup() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_kgroup : g_active_q8_kgroup + +// the q51 repack for a bake whose target has grp q51 planes: some LOCAL registrations +// leave repack_q51 unset (x64-gen keeps disk-order q51 slots), so the gen module registers its +// generic q51 repack here for the override path to reach +var private g_bake_q51_repack = @@repack_q51_identity + +def set_bake_q51_repack(f : RepackQ51Fn) { + g_bake_q51_repack = f +} + +//! The q51 repack the LOAD path invokes — the gen generic under a cross-box override with a +//! grp q51 target, else the active backend's slot. +def active_repack_q51() : RepackQ51Fn { + return g_bake_cpu_override && g_bake_cpu.q51_mr > 1l ? g_bake_q51_repack : g_repack_q51 +} [unused_argument(yp, wp, sp, xqp, xsp, n, d)] def private q8q8_unset_kernel(var yp : float?; wp : int8 const?; sp : float const?; xqp : int8 const?; xsp : float const?; n, d : int64) { no_backend() } [unused_argument(yp, wp, sp, xqp, xsp, n, d, ntok)] diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 74b8c0d048..a1c9ff0b59 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -569,11 +569,11 @@ def repack_q8q8_grp(var wp : int8?; var sp : float?; n, d, mr : int64; wbias : i delete ts } -// the backend repack slot: layout/wbias/kgroup companions decide interleave, plane bias, and -// k-byte grouping — so a bias128-mr16 stamp gets biased grp16 weights and an smmla stamp gets -// the kg8 row-pair plane from the same load path that gives grp4 its plain weights +// the backend repack slot. Numbers come through math's active_* getters — value-identical to +// the stamped companions during normal serving (activation caches from them), and the +// cross-box bake override's one seam def private repack_q8q8_gen(var wp : int8?; var sp : float?; n, d : int64) { - repack_q8q8_grp(wp, sp, n, d, int64(q8q8_layout_gen()), int64(q8q8_wbias_gen()), int64(q8q8_kgroup_gen())) + repack_q8q8_grp(wp, sp, n, d, active_q8_layout_mr(), active_q8_wbias(), active_q8_kgroup()) } //! mr-generalized MXFP4 plane repack (mr=4 reproduces the old hand arm64-laneq mx interleave @@ -626,7 +626,7 @@ def repack_mx4_grp(var np : uint8?; var ep : uint8?; n, d, mr : int64) { // the backend mx4 repack slot: same layout companion as the Q8 repack — planes and weights // interleave at the ONE stamped mr def private repack_mx4_gen(var np : uint8?; var ep : uint8?; n, d : int64) { - repack_mx4_grp(np, ep, n, d, int64(q8q8_layout_gen())) + repack_mx4_grp(np, ep, n, d, active_q8_layout_mr()) } // ===== the q51 GEMV family (self-anchored — q51 does NOT ride the Q8 mr; its grid is @@ -691,9 +691,9 @@ def repack_q51_grp(var qp : uint8?; var sp : uint8?; n, d, mr : int64) { delete ts } -// the backend q51 repack slot: the family's OWN layout companion decides the interleave +// the backend q51 repack slot: the family's layout number via math's override-aware getter def private repack_q51_gen(var qp : uint8?; var sp : uint8?; n, d : int64) { - repack_q51_grp(qp, sp, n, d, int64(q51q8_layout_gen())) + repack_q51_grp(qp, sp, n, d, active_q51_layout_mr()) } //! One row's dot off the grp q51 planes, scalar — the q51 stub's reference body and the @@ -1136,13 +1136,13 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr // the backend kq repack slot: same layout companion as the Q8/mx4 repacks — one stamped mr def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : int64) { if (fmt == 4) { - repack_k4_grp(kq, ks, n, d, int64(k4q8_layout_gen())) + repack_k4_grp(kq, ks, n, d, active_kq_layout_mr(4)) } elif (fmt == 5) { - repack_k5_grp(kq, ks, n, d, int64(k5q8_layout_gen())) + repack_k5_grp(kq, ks, n, d, active_kq_layout_mr(5)) } elif (fmt == 40) { - repack_q40_grp(kq, ks, n, d, int64(q40q8_layout_gen())) + repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(40)) } else { - repack_k6_grp(kq, ks, n, d, int64(k6q8_layout_gen())) + repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } } @@ -2109,6 +2109,9 @@ def private q8q8_groupn_s16_gen(var yp : float?; wp : int8 const?; sp : uint16 c [init] def dasllama_math_gen_register() { + // the cross-box override's q51 reach: x64-gen keeps disk-order q51 slots, so the generic + // grp q51 repack registers here for bakes whose TARGET has grp q51 planes + set_bake_q51_repack(@@repack_q51_gen) // needs_repack keeps this out of auto-select (direct callers stay on arm64-sdot). q8_layout // must be evaluated at selection time, not [init] — [init]s run before the JIT installs // generated code (the slice C mr8 bug); arm64 needs no availability predicate. diff --git a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das index fc7b11b881..84eaa8cf4c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das @@ -6,6 +6,7 @@ module dasllama_math_vulkan shared public require dasllama/dasllama_math require dasllama/dasllama_gemm_schema // kq_qsb/kq_ssb + the q8/kq plane consts — the stride source require dasllama/dasllama_env // the shared typed env readers + the knob registry +require dasllama/dasllama_config // DlimConfiguration — this module registers the vulkan source require dasllama/dasllama_par require dasllama/dasllama_vulkan_lens require vulkan @@ -3993,6 +3994,8 @@ struct private GpuState { has_coopmat2 : bool // device supports NV_cooperative_matrix2 (tensor-addressed wg tiles + decode fns) coopmat_mode : int // Q8_0 prefill GEMM: 0 = sdot4, 1 = f16 coopmat, 2 = int8 coopmat, 3 = mul_mm L-tile (default), 4 = cm2 decode-in-load weight_budget : int64 // resident-weight cap — queried heap budget minus reserve, or VRAM_BUDGET + dry : bool // OFFLINE BAKE: accept/refuse arithmetic runs, every device call is gated off + msr : int64 // maxStorageBufferRange — device-queried at init, config-supplied when dry @do_not_delete stacks : array // the resident driver's weight arenas, one per format present (see the arena section). Separate // from `stacks` on purpose: the two tiers never co-arm on one model, and the arena has no @@ -4197,7 +4200,7 @@ def private find_host_mem(type_bits : uint; cached : bool) : uint { return find_memory_type(g_gpu.phys, type_bits, want) } -def private make_host_buf(bytes : int64; storage : bool; cached : bool = false; xfer_shared : bool = false) : HostBuf { +def private make_host_buf(bytes : int64; storage : bool; cached : bool = false; xfer_shared : bool = false; soft : bool = false) : HostBuf { var bci : BufferCreateInfo bci.size = uint64(bytes) bci.usage.storage_buffer = storage @@ -4212,7 +4215,19 @@ def private make_host_buf(bytes : int64; storage : bool; cached : bool = false; let req = get_buffer_memory_requirements(g_gpu.device, b) let mai = MemoryAllocateInfo(allocationSize = req.size, memoryTypeIndex = find_host_mem(req.memoryTypeBits, cached)) - var m <- allocate_memory(g_gpu.device, mai) + var ares = VkResult.SUCCESS + var m <- allocate_memory(g_gpu.device, mai, unsafe(addr(ares))) + if (ares != VkResult.SUCCESS) { + // soft callers (stream mirrors) degrade — the walk treats it as its budget stop + if (!soft) { + panic("dasLLAMA vulkan tier: pinned host alloc failed ({ares}, {bytes} bytes, type {mai.memoryTypeIndex})") + } + to_log(LOG_INFO, "dasLLAMA vulkan tier: pinned mirror alloc declined ({ares}, {bytes} bytes, type {mai.memoryTypeIndex}) — streamed walk stops here\n") + unsafe { + vkDestroyBuffer(dev_raw(), reinterpret(b._vk), null) + } + return HostBuf() + } bind_buffer_memory(g_gpu.device, b, m, 0ul) var mapped : void? = null let mf : VkMemoryMapFlags @@ -4302,6 +4317,151 @@ def private nonowning_buf(h : uint64) : Buffer { return b } +// The coopmat-mode ladder, shared by vk_moe_init and the DlimConfiguration source — one +// resolver, so the config's recorded mode can never drift from the mode the device runs. +def private resolve_coopmat_mode(has_cm, has_cm2 : bool) : int { + // fastest MEASURED default (mm; cm2 opt-in until it beats mm); DASLLAMA_COOPMAT overrides + var cmode = has_cm ? 3 : 0 + if (has_cm) { + let mode = env_str("DASLLAMA_COOPMAT", "") + if (mode == "sdot4") { + cmode = 0 + } elif (mode == "f16") { + cmode = 1 + } elif (mode == "int8") { + cmode = 2 + } elif (mode == "mm") { + cmode = 3 + } elif (mode == "cm2" && has_cm2) { + cmode = 4 + } + } + if (g_coopmat_mode_force >= 0) { + cmode = has_cm ? g_coopmat_mode_force : 0 + } + if (cmode == 4 && !has_cm2) { + cmode = 3 // forced/requested cm2 without the extension: mm + } + return cmode +} + +struct private VkHeapInfo { + budget : int64 // largest device-local heap's current budget (0 = no VK_EXT_memory_budget) + used : int64 + size : int64 +} + +def private query_heap_info(phys : VkPhysicalDevice) : VkHeapInfo { + var r = VkHeapInfo() + // heap SIZE needs no extension — only the live budget/usage ride VK_EXT_memory_budget + let has_budget = device_extension_available(phys, "VK_EXT_memory_budget") + var mb = VkPhysicalDeviceMemoryBudgetPropertiesEXT() // sType pre-filled by the ctor + var mp2 = VkPhysicalDeviceMemoryProperties2() + if (has_budget) { + unsafe { + mp2.pNext = addr(mb) + } + } + vkGetPhysicalDeviceMemoryProperties2(phys, mp2) + var best = -1 + for (i in range(int(mp2.memoryProperties.memoryHeapCount))) { + if (!mp2.memoryProperties.memoryHeaps[i].flags.device_local) { + continue + } + // rank by live budget when the ext reports it, by raw size otherwise + if (best < 0) { + best = i + } elif (has_budget ? mb.heapBudget[i] > mb.heapBudget[best] + : mp2.memoryProperties.memoryHeaps[i].size > mp2.memoryProperties.memoryHeaps[best].size) { + best = i + } + } + if (best >= 0) { + r.size = int64(mp2.memoryProperties.memoryHeaps[best].size) + if (has_budget && mb.heapBudget[best] > 0ul) { + r.budget = int64(mb.heapBudget[best]) + r.used = int64(mb.heapUsage[best]) + } + } + return r +} + +// The pre-carve weight cap, shared by vk_moe_init and the DlimConfiguration source. Env wins +// outright (=0 pins WEIGHT_CAP — the misreporting-driver hatch); auto derives from heap SIZE, a +// hardware constant — same number every run. Transient pressure is the memprio shield's job. +def private resolve_weight_cap(heap : VkHeapInfo) : int64 { + let vram_mb = gpu_want_vram_mb() + if (vram_mb > 0l) { + return vram_mb * 1_000_000l + } + if (has_env_variable("DASLLAMA_GPU_VRAM_MB")) { + return WEIGHT_CAP + } + var cap = WEIGHT_CAP + if (heap.size > 0l) { + cap = clamp(heap.size - VRAM_RESERVE, 0l, cap) + } + return cap +} + +// The stream-slot carve, derived AT READ TIME — g_gpu.weight_budget stores the UNCARVED cap. +// Init-time carving was timing-dependent (the expert_q8n probe can init the device BEFORE the +// loader reports its slot need), which made a dry bake's plan diverge from the live walk's. +def private carved_budget : int64 { + var r = g_gpu.weight_budget + if (gpu_want_moe_stream() != 0l) { + let need = moe_gpu_stream_need() + if (need > 0l) { + r -= 2l * need + } elif (gpu_want_moe_stream() > 0l) { + r -= STREAM_RESERVE + } + } + return max(r, 0l) +} + +var private g_vk_dry_want = false +var private g_vk_dry_cfg = DlimVulkanConfig() + +// fill the env-else-programmatic wants half of a config section (the caps half is the +// device's / the dry config's) — shared by the live source and the dry arm's verify +def private vk_dlim_wants_fill(var v : DlimVulkanConfig) { + v.moe_layers = gpu_want_moe_layers() + v.moe_stream = gpu_want_moe_stream() + v.dn = gpu_want_dn() + v.attn = gpu_want_attn() + v.dnd = gpu_want_dnd() + v.shexp = gpu_want_shexp() + v.qkv = gpu_want_qkv() + v.cls = gpu_want_cls_upload() // the WALK's gate, not the arm-intent (they had split defaults) + v.dense = gpu_want_dense() + v.dense_attn = gpu_want_dense_attn() + v.dense_shexp = gpu_want_dense_shexp() + v.trim = env_flag("DASLLAMA_TRIM", false) +} + +//! Arm the tier for an OFFLINE bake: the load walk runs its full accept/refuse arithmetic +//! against the config's numbers, gathers collect into the bake blob, and no Vulkan call is +//! made — no instance, no device, no VRAM. The converter's `-f vulkan` rail. +def vulkan_dry_bake_arm(v : DlimVulkanConfig) : bool { + // the walk's gates read env-else-programmatic wants — a bake whose config disagrees with + // them would write an image whose identity lies about its own plan. Fail closed, name both. + var w = v + vk_dlim_wants_fill(w) + w.trim = v.trim // trim shapes the SAVE, not the walk — the converter's flag decides it + if (dlim_vulkan_tag(w) != dlim_vulkan_tag(v)) { + to_log(LOG_ERROR, "dasLLAMA vulkan tier: dry-bake config disagrees with this process' wants (set the DASLLAMA_GPU_* knobs to match)\n config {dlim_vulkan_tag(v)}\n wants {dlim_vulkan_tag(w)}\n") + return false + } + g_vk_dry_want = true + g_vk_dry_cfg = v + set_moe_gpu_layer_count(v.moe_layers) + if (!moe_gpu_tier_installed()) { + install_moe_gpu_tier(@@vk_moe_ffn, @@vk_moe_upload_stack, @@vk_moe_cls, @@vk_moe_dense, @@vk_moe_drop_stacks, @@vk_moe_dn, @@vk_moe_attn) + } + return true +} + def private vk_moe_init : bool { if (g_init_failed) { // checked FIRST: a partially-built g_gpu must never report ready return false @@ -4309,6 +4469,24 @@ def private vk_moe_init : bool { if (g_gpu != null) { return true } + if (g_vk_dry_want) { + // DlimVulkanConfig numbers feed the SAME arithmetic — a dry bake cannot drift from live + if (g_vk_dry_cfg.subgroup_size < 32) { + to_log(LOG_ERROR, "dasLLAMA vulkan tier: config subgroupSize {g_vk_dry_cfg.subgroup_size} < 32 — bake declined\n") + g_init_failed = true + return false + } + g_gpu = new GpuState() + g_gpu.dry = true + g_gpu.coopmat_mode = g_vk_dry_cfg.coopmat_mode + g_gpu.has_coopmat = g_vk_dry_cfg.coopmat_mode > 0 + g_gpu.has_coopmat2 = g_vk_dry_cfg.coopmat_mode == 4 + g_gpu.rows_per_wg = int64((WG_X + uint(g_vk_dry_cfg.subgroup_size) - 1u) / uint(g_vk_dry_cfg.subgroup_size)) + g_gpu.msr = g_vk_dry_cfg.max_storage_range + g_gpu.weight_budget = g_vk_dry_cfg.vram_mb * 1_000_000l + + return true + } g_init_failed = true // cleared only on full success if (volkInitialize() != 0) { to_log(LOG_ERROR, "dasLLAMA vulkan tier: no Vulkan loader\n") @@ -4327,28 +4505,7 @@ def private vk_moe_init : bool { // coopmat requested whenever supported — the fa twin needs it even in sdot4 GEMM mode g_gpu.has_coopmat = cooperative_matrix_supported(g_gpu.phys) g_gpu.has_coopmat2 = g_gpu.has_coopmat && cooperative_matrix2_supported(g_gpu.phys) - // fastest MEASURED default (mm; cm2 opt-in until it beats mm); DASLLAMA_COOPMAT overrides - g_gpu.coopmat_mode = g_gpu.has_coopmat ? 3 : 0 - if (g_gpu.has_coopmat) { - let mode = env_str("DASLLAMA_COOPMAT", "") - if (mode == "sdot4") { - g_gpu.coopmat_mode = 0 - } elif (mode == "f16") { - g_gpu.coopmat_mode = 1 - } elif (mode == "int8") { - g_gpu.coopmat_mode = 2 - } elif (mode == "mm") { - g_gpu.coopmat_mode = 3 - } elif (mode == "cm2" && g_gpu.has_coopmat2) { - g_gpu.coopmat_mode = 4 - } - } - if (g_coopmat_mode_force >= 0) { - g_gpu.coopmat_mode = g_gpu.has_coopmat ? g_coopmat_mode_force : 0 - } - if (g_gpu.coopmat_mode == 4 && !g_gpu.has_coopmat2) { - g_gpu.coopmat_mode = 3 // forced/requested cm2 without the extension: mm - } + g_gpu.coopmat_mode = resolve_coopmat_mode(g_gpu.has_coopmat, g_gpu.has_coopmat2) // dedicated transfer family: streamed copies overlap compute (DASLLAMA_VK_XFERQ=0 = bisect hatch) var xfam = -1 if (g_gpu.has_coopmat && env_flag("DASLLAMA_VK_XFERQ", true) @@ -4405,49 +4562,22 @@ def private vk_moe_init : bool { return false } g_gpu.rows_per_wg = int64((WG_X + sg - 1u) / sg) // = gl_NumSubgroups for our 1D workgroup - // env wins outright (=0 pins WEIGHT_CAP — the misreporting-driver hatch); else live budget - reserve - g_gpu.weight_budget = WEIGHT_CAP - let vram_mb = gpu_want_vram_mb() - let vram_override = vram_mb > 0l || has_env_variable("DASLLAMA_GPU_VRAM_MB") - if (vram_mb > 0l) { - g_gpu.weight_budget = vram_mb * 1_000_000l - } - if (device_extension_available(g_gpu.phys, "VK_EXT_memory_budget")) { - var mb = VkPhysicalDeviceMemoryBudgetPropertiesEXT() // sType pre-filled by the ctor - var mp2 = VkPhysicalDeviceMemoryProperties2() - unsafe { - mp2.pNext = addr(mb) - } - vkGetPhysicalDeviceMemoryProperties2(g_gpu.phys, mp2) - var best = -1 - for (i in range(int(mp2.memoryProperties.memoryHeapCount))) { - if (mp2.memoryProperties.memoryHeaps[i].flags.device_local - && (best < 0 || mb.heapBudget[i] > mb.heapBudget[best])) { - best = i - } - } - if (best >= 0 && mb.heapBudget[best] > 0ul) { - let safe_cap = max(int64(mb.heapBudget[best]) - VRAM_RESERVE, 0l) - // the honest cliff leaves 2GB of the raw heap for the desktop (6000MB healthy, 6803 demoted) - let cliff = min(safe_cap, int64(mp2.memoryProperties.memoryHeaps[best].size) - 2_147_483_648l) - if (!vram_override) { - g_gpu.weight_budget = min(safe_cap, g_gpu.weight_budget) - } elif (g_gpu.weight_budget > cliff) { - to_log(LOG_WARNING, "dasLLAMA vulkan tier: DASLLAMA_GPU_VRAM_MB {g_gpu.weight_budget / 1000000l} MB is above the demotion cliff — WDDM will silently demote buffers to SysMem (act-role collapse); lower it to <= {cliff / 1000000l} MB\n") - } - to_log(LOG_INFO, "dasLLAMA vulkan tier: device heap budget {int64(mb.heapBudget[best]) / 1000000l} MB (used {int64(mb.heapUsage[best]) / 1000000l}), weight cap {g_gpu.weight_budget / 1000000l} MB\n") - } - } - // stream-slot carve: exact loader-reported need, else the legacy constant (direct-upload paths) - if (gpu_want_moe_stream() != 0l) { - let need = moe_gpu_stream_need() - if (need > 0l) { - g_gpu.stream_reserve = 2l * need - } elif (gpu_want_moe_stream() > 0l) { - g_gpu.stream_reserve = STREAM_RESERVE - } - g_gpu.weight_budget = max(g_gpu.weight_budget - g_gpu.stream_reserve, 0l) + var pdp : VkPhysicalDeviceProperties + vkGetPhysicalDeviceProperties(g_gpu.phys, pdp) + g_gpu.msr = int64(pdp.limits.maxStorageBufferRange) + // env-vs-auto resolution shared with the DlimConfiguration source (resolve_weight_cap) + let heap = query_heap_info(g_gpu.phys) + g_gpu.weight_budget = resolve_weight_cap(heap) + if (heap.budget > 0l) { + let vram_override = gpu_want_vram_mb() > 0l || has_env_variable("DASLLAMA_GPU_VRAM_MB") + // the honest cliff leaves 2GB of the raw heap for the desktop (6000MB healthy, 6803 demoted) + let cliff = clamp(heap.budget - VRAM_RESERVE, 0l, heap.size - 2_147_483_648l) + if (vram_override && g_gpu.weight_budget > cliff) { + to_log(LOG_WARNING, "dasLLAMA vulkan tier: DASLLAMA_GPU_VRAM_MB {g_gpu.weight_budget / 1000000l} MB is above the demotion cliff — WDDM will silently demote buffers to SysMem (act-role collapse); lower it to <= {cliff / 1000000l} MB\n") + } + to_log(LOG_INFO, "dasLLAMA vulkan tier: device heap budget {heap.budget / 1000000l} MB (used {heap.used / 1000000l}), weight cap {g_gpu.weight_budget / 1000000l} MB\n") } + // ReBAR: DL|HV|HC on the full-VRAM heap (the >1GB check excludes the 256MiB legacy window) if (env_flag("DASLLAMA_VK_REBAR", true)) { var rmp : VkPhysicalDeviceMemoryProperties @@ -4632,7 +4762,8 @@ def private stack_plane_bytes(n, rows : int64; fmt : int) : tuple push(st) return length(g_gpu.stacks) - 1 } @@ -4666,17 +4797,163 @@ def private ensure_gemv_scratch(si : int) { g_gpu.stacks[si].gemv_made = true } -// stream arm: keep the gathered device-layout planes in a pinned host mirror (host-visible -// memory the copy engine DMA-streams into a slot per prefill). No VRAM spent here. +// import support is spec-gray per driver: one decline disables imports for the process (say WHY +// once) — every later stack goes straight to the pinned copy instead of churning failed vk calls +def private vk_import_declined(why : string) { + if (!g_vk_import_declined_said) { + g_vk_import_declined_said = true + to_log(LOG_INFO, "dasLLAMA vulkan tier: mapped-image import declined ({why}) — pinned mirror copies\n") + } +} + +// P2: import a 4KB-aligned mapped-file window as a transfer_src buffer (VK_EXT_external_memory_host). +// buf == 0 = declined, the caller falls back to the pinned copy. NVIDIA/Windows refuses FILE-BACKED +// pages in BOTH protections while anon imports succeed (full 2x2: _vk_import_probe.das, 2026-07-28) +def private make_imported_buf(ptr : void?; bytes : int64) : HostBuf { + let hb = HostBuf() + if (ptr == null || bytes <= 0l || !device_extension_available(g_gpu.phys, "VK_EXT_external_memory_host")) { + return hb + } + let algn = int64(external_memory_host_min_alignment(g_gpu.phys)) + if (algn <= 0l || int64(intptr(ptr)) % algn != 0l) { + return hb + } + // the round-up stays inside the mapping: windows start 4KB-aligned, the file is 16KB-padded + let sz = (bytes + algn - 1l) / algn * algn + var ht : VkExternalMemoryHandleTypeFlags + ht.host_allocation_ext = true + var hpp = VkMemoryHostPointerPropertiesEXT() + let hres = vkGetMemoryHostPointerPropertiesEXT(dev_raw(), ht, ptr, hpp) + if (hres != VkResult.SUCCESS) { + vk_import_declined("vkGetMemoryHostPointerPropertiesEXT {hres}") + return hb + } + var emb = VkExternalMemoryBufferCreateInfo() + emb.handleTypes = ht + var bci : BufferCreateInfo + bci.size = uint64(sz) + bci.usage.transfer_src = true + if (g_gpu.xfer_fam >= 0) { + // same CONCURRENT sharing the pinned mirrors use (transfer-queue fills + compute heat slices) + bci.sharingMode = VkSharingMode.CONCURRENT + bci.pQueueFamilyIndices <- [g_gpu.fam, uint(g_gpu.xfer_fam)] + } + unsafe { + bci.next = addr(emb) + } + var cres = VkResult.SUCCESS + var b <- create_buffer(g_gpu.device, bci, unsafe(addr(cres))) // deliberately never finalized (make_host_buf's rule) + if (cres != VkResult.SUCCESS) { + vk_import_declined("vkCreateBuffer {cres}") + return hb + } + let req = get_buffer_memory_requirements(g_gpu.device, b) + let usable = hpp.memoryTypeBits & req.memoryTypeBits + if (usable == 0u || int64(req.size) > sz) { + unsafe { + vkDestroyBuffer(dev_raw(), reinterpret(b._vk), null) + } + vk_import_declined(usable == 0u ? "no usable memory type" : "buffer wants {req.size} bytes over a {sz} byte window") + return hb + } + // prefer host_cached NON-device-local: the lowest usable bit can be the ReBAR heap (multi-GB imports exhaust VRAM), and imported file pages are cached RAM — an attribute mismatch is a driver refusal + var mp : VkPhysicalDeviceMemoryProperties + vkGetPhysicalDeviceMemoryProperties(g_gpu.phys, mp) + var ti = 0u + while ((usable & (1u << ti)) == 0u) { + ti++ + } + var plain = -1 + var cached = -1 + for (i in range(int(mp.memoryTypeCount))) { + let fl = mp.memoryTypes[i].propertyFlags + if ((usable & (1u << uint(i))) != 0u && fl.host_visible && !fl.device_local) { + plain = plain < 0 ? i : plain + cached = cached < 0 && fl.host_cached ? i : cached + } + } + if (cached >= 0) { + ti = uint(cached) + } elif (plain >= 0) { + ti = uint(plain) + } + var imp = VkImportMemoryHostPointerInfoEXT() + imp.handleType = ht + imp.pHostPointer = ptr + var mai = MemoryAllocateInfo(allocationSize = uint64(sz), memoryTypeIndex = ti) + unsafe { + mai.next = addr(imp) + } + var ares = VkResult.SUCCESS + var m <- allocate_memory(g_gpu.device, mai, unsafe(addr(ares))) + if (ares != VkResult.SUCCESS) { + unsafe { + vkDestroyBuffer(dev_raw(), reinterpret(b._vk), null) + } + vk_import_declined("vkAllocateMemory {ares}, type {ti} of usable {usable} ({mp.memoryTypes[int(ti)].propertyFlags}), {sz} bytes") + return hb + } + var bres = VkResult.SUCCESS + bind_buffer_memory(g_gpu.device, b, m, 0ul, unsafe(addr(bres))) + if (bres != VkResult.SUCCESS) { + unsafe { + vkFreeMemory(dev_raw(), reinterpret(m._vk), null) + vkDestroyBuffer(dev_raw(), reinterpret(b._vk), null) + } + vk_import_declined("vkBindBufferMemory {bres}") + return hb + } + g_gpu.host_mem |> insert(b._vk, m._vk) + return HostBuf(buf = b._vk, mem = m._vk, mapped = ptr, bytes = bytes) +} + +// stream arm: pinned host mirror the copy engine DMAs into a slot per prefill (no VRAM spent); P2: a flavor load IMPORTS the gather's mapped-blob window instead — zero pinned RAM, the page cache is the mirror def private vk_moe_stream_stack(wq : uint8 const?; ws : uint8 const?; woff : int64; n, rows : int64; fmt : int) : bool { let pb = stack_plane_bytes(n, rows, fmt) - var ss = StreamStack(base = woff, elems = rows * n, fmt = fmt, wqbytes = pb.wq, wsbytes = pb.ws, - hwq = make_host_buf(pb.wq, false, [xfer_shared = true]), - hws = make_host_buf(max(pb.ws, 64l), false, [xfer_shared = true])) - unsafe { - par_memcpy(ss.hwq.mapped, reinterpret(wq), pb.wq) - if (pb.ws > 0l) { // q8n mirrors carry no scale plane - par_memcpy(ss.hws.mapped, reinterpret(ws), pb.ws) + if (env_flag("DASLLAMA_VK_IMPORT", true) && !g_vk_import_declined_said) { + let win = moe_gpu_slice_window(woff, fmt) + if (win.wq != null && win.wqb == pb.wq && (pb.ws <= 0l || (win.ws != null && win.wsb == pb.ws))) { + var iq = make_imported_buf(win.wq, pb.wq) + if (iq.buf != 0ul) { + // ws: import the real window, or a tiny pinned dummy for the descriptor floor (q8n) + let is_ = pb.ws > 0l ? make_imported_buf(win.ws, pb.ws) : make_host_buf(64l, false, [xfer_shared = true]) + if (is_.buf != 0ul) { + var ss = StreamStack(base = woff, elems = rows * n, fmt = fmt, + wqbytes = pb.wq, wsbytes = pb.ws, hwq = iq, hws = is_) + if (length(g_gpu.stream_stacks) % 3 == 2) { + g_gpu.stream_max_dn_wq = max(g_gpu.stream_max_dn_wq, pb.wq) + g_gpu.stream_max_dn_ws = max(g_gpu.stream_max_dn_ws, pb.ws) + } else { + g_gpu.stream_max_gu_wq = max(g_gpu.stream_max_gu_wq, pb.wq) + g_gpu.stream_max_gu_ws = max(g_gpu.stream_max_gu_ws, pb.ws) + } + g_gpu.stream_stacks |> emplace(ss) + if (!g_vk_import_said) { + g_vk_import_said = true + to_log(LOG_INFO, "dasLLAMA vulkan tier: stream mirrors IMPORT the mapped image (external_memory_host) — zero pinned copies\n") + } + return true + } + destroy_host_buf(iq) + } + } + } + var ss = StreamStack(base = woff, elems = rows * n, fmt = fmt, wqbytes = pb.wq, wsbytes = pb.ws) + if (!g_gpu.dry) { + ss.hwq = make_host_buf(pb.wq, false, [xfer_shared = true, soft = true]) + if (ss.hwq.buf == 0ul) { + return false // pinned-mirror stop: the walk rolls the partial triple back and breaks + } + ss.hws = make_host_buf(max(pb.ws, 64l), false, [xfer_shared = true, soft = true]) + if (ss.hws.buf == 0ul) { + destroy_host_buf(ss.hwq) + return false + } + unsafe { + par_memcpy(ss.hwq.mapped, reinterpret(wq), pb.wq) + if (pb.ws > 0l) { // q8n mirrors carry no scale plane + par_memcpy(ss.hws.mapped, reinterpret(ws), pb.ws) + } } } if (length(g_gpu.stream_stacks) % 3 == 2) { // walk order gate/up/down: this one is a down @@ -4690,6 +4967,27 @@ def private vk_moe_stream_stack(wq : uint8 const?; ws : uint8 const?; woff : int return true } +// The resident-accept arithmetic, shared by the upload path and the slice replay's would-accept +// probe — a divergent copy would desync the baked plan at the refusal boundary. +// 0 = accept, 1 = VRAM budget, 2 = descriptor capacity. +def private vk_moe_resident_verdict(n, rows : int64; fmt : int) : int { + let pb = stack_plane_bytes(n, rows, fmt) + if (g_gpu.resident_bytes + pb.wq + pb.ws > carved_budget()) { + return 1 + } + return length(g_gpu.stacks) >= MAX_STACKS ? 2 : 0 +} + +//! MoeGpuWouldAcceptFn: would the tier accept this stack right now? Pure arithmetic, no side +//! effects — the slice replay uses it to distinguish a refusal boundary (the walk attempts a +//! gather the bake refused, so it is NOT in the plan) from real plan divergence. +def vk_moe_would_accept(n : int64; rows : int64; fmt : int; stream : bool) : bool { + if (g_gpu == null) { + return true // tier not up: claim accept so a mismatch reads as real divergence + } + return stream || vk_moe_resident_verdict(n, rows, fmt) == 0 +} + //! MoeGpuUploadFn: upload one weight stack ([rows x n] at element offset woff) from the loader's //! gathered row-major planes. stream = pinned host mirror for per-prefill DMA instead of VRAM. def vk_moe_upload_stack(wq : uint8 const?; ws : uint8 const?; woff : int64; n : int64; rows : int64; fmt : int; stream : bool) : bool { @@ -4699,20 +4997,23 @@ def vk_moe_upload_stack(wq : uint8 const?; ws : uint8 const?; woff : int64; n : if (stream) { return vk_moe_stream_stack(wq, ws, woff, n, rows, fmt) } - let pb = stack_plane_bytes(n, rows, fmt) - let wqbytes = pb.wq - let wsbytes = pb.ws - if (g_gpu.resident_bytes + wqbytes + wsbytes > g_gpu.weight_budget) { + let verdict = vk_moe_resident_verdict(n, rows, fmt) + if (verdict == 1) { to_log(LOG_INFO, "dasLLAMA vulkan tier: VRAM budget reached at {g_gpu.resident_bytes} bytes\n") return false } - if (length(g_gpu.stacks) >= MAX_STACKS) { + if (verdict == 2) { to_log(LOG_INFO, "dasLLAMA vulkan tier: descriptor capacity reached at {length(g_gpu.stacks)} stacks\n") return false } + let pb = stack_plane_bytes(n, rows, fmt) + let wqbytes = pb.wq + let wsbytes = pb.ws let si = make_stack_shell(woff, rows * n, fmt, wqbytes, wsbytes) - upload_region(g_gpu.stacks[si].wqbuf, wq, wqbytes) - upload_region(g_gpu.stacks[si].wsbuf, ws, wsbytes) + if (!g_gpu.dry) { + upload_region(g_gpu.stacks[si].wqbuf, wq, wqbytes) + upload_region(g_gpu.stacks[si].wsbuf, ws, wsbytes) + } g_gpu.resident_bytes += wqbytes + wsbytes return true } @@ -4723,8 +5024,10 @@ def vk_moe_drop_stacks(n : int; stream : bool) { if (stream) { for (_i in range(n)) { let k = length(g_gpu.stream_stacks) - 1 - destroy_host_buf(g_gpu.stream_stacks[k].hwq) - destroy_host_buf(g_gpu.stream_stacks[k].hws) + if (!g_gpu.dry) { + destroy_host_buf(g_gpu.stream_stacks[k].hwq) + destroy_host_buf(g_gpu.stream_stacks[k].hws) + } g_gpu.stream_stacks |> resize(k) } return @@ -4734,8 +5037,10 @@ def vk_moe_drop_stacks(n : int; stream : bool) { assert(!g_gpu.stacks[k].gemv_made && !g_gpu.stacks[k].batch_made && !g_gpu.stacks[k].cls_made, "dasLLAMA vulkan tier: dropping a stack with live dispatch state") g_gpu.resident_bytes -= g_gpu.stacks[k].wqbytes + g_gpu.stacks[k].wsbytes - destroy_device_buf(g_gpu.stacks[k].wqbuf) - destroy_device_buf(g_gpu.stacks[k].wsbuf) + if (!g_gpu.dry) { + destroy_device_buf(g_gpu.stacks[k].wqbuf) + destroy_device_buf(g_gpu.stacks[k].wsbuf) + } g_gpu.stacks |> resize(k) } } @@ -4747,6 +5052,14 @@ def vk_drop_model_state { if (g_gpu == null) { return } + if (g_gpu.dry) { + // dry bake owns no device objects — reset the accounting shells only + g_gpu.stacks |> clear() + g_gpu.stream_stacks |> clear() + g_gpu.arenas |> clear() + g_gpu.resident_bytes = 0l + return + } vk_check(vkQueueWaitIdle(g_gpu.queue), null) if (g_gpu.xfer_fam >= 0 && g_gpu.xfer_value > 0ul) { xfer_host_wait(g_gpu.xfer_value) // in-flight transfer-queue copies retire before the sweeps @@ -4928,15 +5241,13 @@ def private arena_reserve(fmt : int; cap_blocks : int64) : bool { // q8n has no scale plane — floor the ws buffer so its descriptor slots stay valid (stack-shell rule) let sbytes = max(cap_blocks * bb.ws, 64l) // maxStorageBufferRange is the real ceiling (uint32 indexing reaches 16GiB); NV/AMD report 4GiB - var lp : VkPhysicalDeviceProperties - vkGetPhysicalDeviceProperties(g_gpu.phys, lp) - let shard_cap = int64(lp.limits.maxStorageBufferRange) + let shard_cap = g_gpu.msr if (qbytes > shard_cap || sbytes > shard_cap) { to_log(LOG_WARNING, "dasLLAMA vulkan arena: fmt {fmt} needs {qbytes} quant + {sbytes} scale bytes — over the device's {shard_cap}B storage-range cap\n") return false } var a = ArenaFmt(fmt = fmt, cap_blocks = cap_blocks, wqbytes = qbytes, wsbytes = sbytes, - wqbuf = make_device_buf(qbytes), wsbuf = make_device_buf(sbytes)) + wqbuf = g_gpu.dry ? 0ul : make_device_buf(qbytes), wsbuf = g_gpu.dry ? 0ul : make_device_buf(sbytes)) g_gpu.arenas |> insert(fmt, a) return true } @@ -4956,9 +5267,11 @@ def private arena_place(wq : uint8 const?; ws : uint8 const?; n, rows : int64; f let blk = a.blocks a.blocks += need let bb = arena_block_bytes(fmt) - upload_region_at(a.wqbuf, blk * bb.wq, wq, need * bb.wq) - if (bb.ws > 0l) { - upload_region_at(a.wsbuf, blk * bb.ws, ws, need * bb.ws) + if (!g_gpu.dry) { + upload_region_at(a.wqbuf, blk * bb.wq, wq, need * bb.wq) + if (bb.ws > 0l) { + upload_region_at(a.wsbuf, blk * bb.ws, ws, need * bb.ws) + } } return blk } @@ -5428,6 +5741,9 @@ def vk_rdec_prepare(n_layers, dim, qd, kv_dim, head_size, n_heads, hidden, vocab to_log(LOG_WARNING, "dasLLAMA vulkan resident: dim {dim} > {AR_MAX_DIM} or head_size {head_size} > 256 exceeds the workgroup row slabs\n") return false } + if (g_gpu.dry) { + return true // guards above ARE the plan decision; everything below is device state + } g_rd = new RDec(ready = false, n_layers = n_layers, dim = dim, qd = qd, kv_dim = kv_dim, head_size = head_size, n_heads = n_heads, kv_mul = n_heads / (kv_dim / head_size), hidden = hidden, vocab = vocab, seq_cap = seq_cap, neox = neox, eps = eps, scale = scale, @@ -5479,6 +5795,9 @@ def vk_rdec_prepare(n_layers, dim, qd, kv_dim, head_size, n_heads, hidden, vocab //! rms_final], dim floats each, exactly the layout the layer w_offs index. Under qk_norm the //! per-head [rms_q[l], rms_k[l]] x L rows (head_size floats each) follow. def vk_rdec_upload_norms(norms : array) { + if (g_gpu != null && g_gpu.dry) { + return + } assert(g_rd != null, "vk_rdec_upload_norms before prepare") upload_region_at(g_rd.norms_dev, 0l, unsafe(addr(norms[0])), long_length(norms) * 4l) // the fused-rail constants: eps/ascale past the norm rows, and (under qk_norm) a copy of the @@ -5505,6 +5824,9 @@ def vk_rdec_upload_norms(norms : array) { //! once per layer after prepare. `w_ffn_off`/`w_next_off` are the rms_ffn[l] and rms_att[l+1] //! (or rms_final for the last layer) offsets in the norms buffer (dim-element units). def vk_rdec_set_layer(l : int64; bq, bk, bv, bo, b1, b3, b2 : int64; fq, fk, fv, fo, f1, f3, f2 : int) { + if (g_gpu != null && g_gpu.dry) { + return + } assert(g_rd != null, "vk_rdec_set_layer before prepare") while (long_length(g_rd.layers) <= l) { g_rd.layers |> emplace(RLayer()) @@ -5623,6 +5945,9 @@ def vk_rdec_set_layer(l : int64; bq, bk, bv, bo, b1, b3, b2 : int64; fq, fk, fv, //! Register the classifier plane (arena block + format) and build the prologue / final-norm / cls //! sets. Call after all layers are set. def vk_rdec_set_cls(cls_block : int64; cls_fmt : int) { + if (g_gpu != null && g_gpu.dry) { + return + } assert(g_rd != null, "vk_rdec_set_cls before prepare") let dim = g_rd.dim let vocab = g_rd.vocab @@ -8420,7 +8745,7 @@ def private heat_pool_make(l : int64; b1, b3, b2 : int64; f1, f3, f2 : int; n, n let pb3 = stack_plane_bytes(n, nfe, f3) let pb2 = stack_plane_bytes(nfe, n, f2) let per_slot = pb1.wq + pb1.ws + pb3.wq + pb3.ws + pb2.wq + pb2.ws - let c = min(want, (g_gpu.weight_budget - g_gpu.resident_bytes) / per_slot) + let c = min(want, (carved_budget() - g_gpu.resident_bytes) / per_slot) if (c <= 0l) { to_log(LOG_INFO, "dasLLAMA vulkan tier: heat cache skips layer {l} (VRAM budget)\n") g_gpu.heat_pools[l] <- pool @@ -8774,6 +9099,31 @@ def vk_moe_cls(var yp : float?; woff : int64; xqp : int8 const?; xsp : float con // ===== registration ===== +var private g_vk_import_said = false // one import proof-line per process +var private g_vk_import_declined_said = false + +// MoeGpuUnmapNotifyFn: a model's mapping is about to unmap — release imported device memory +// over it FIRST (reading through a freed import faults the device). Whole-model teardown: the +// single-owner tier's stream mirrors all belong to the unmapping model when any hit the range. +def private vk_unmap_notify(base : void?; bytes : uint64) { + if (g_gpu == null || g_gpu.dry || base == null) { + return + } + var hit = false + let b0 = intptr(base) + for (ss in g_gpu.stream_stacks) { + let p = intptr(ss.hwq.mapped) + if (p >= b0 && p < b0 + bytes) { + hit = true + break + } + } + if (hit) { + to_log(LOG_INFO, "dasLLAMA vulkan tier: model unmapping — dropping imported stream mirrors first\n") + vk_drop_model_state() + } +} + var private g_vk_probed = false var private g_vk_available = false var private g_vk_why = "" // why the device probe said no — discarded before, which made it silent @@ -8828,7 +9178,7 @@ def private vk_resident_bytes : int64 => g_gpu != null ? g_gpu.resident_bytes : // The resident driver sizes a whole model against these BEFORE uploading anything. Probing the // device here (rather than reporting 0 until something else arms it) is what lets the gate answer // "would this fit?" on a cold tier. -def private vk_weight_budget : int64 => vk_moe_init() ? g_gpu.weight_budget : 0l +def private vk_weight_budget : int64 => vk_moe_init() ? carved_budget() : 0l def private vk_plane_bytes(n, rows : int64; fmt : int) : int64 { let pb = stack_plane_bytes(n, rows, fmt) @@ -8886,18 +9236,49 @@ def vulkan_moe_gpu_arm : bool { return true } -// the vulkan flavor's config identity. ENV-OR-AUTO knobs only — resolving them (vk_moe_init) -// would carve the stream reserve BEFORE the loader reports the model's slot need (518ae164a); -// "auto" resolves deterministically per box and the image is local, so the identity is stable. -def private vk_bake_tag : string { +// Fill the caps-derived half of the vulkan config section from a physical device. Phys-level +// queries only — vk_moe_init stays untouched, so the stream-reserve carve still happens AFTER +// the loader reports the model's slot need (518ae164a). +def private vk_caps_fill(phys : VkPhysicalDevice; var v : DlimVulkanConfig) { + let has_cm = cooperative_matrix_supported(phys) + let has_cm2 = has_cm && cooperative_matrix2_supported(phys) + v.coopmat_mode = resolve_coopmat_mode(has_cm, has_cm2) + v.expert_q8n = v.coopmat_mode == 4 + v.subgroup_size = int(subgroup_properties(phys).subgroupSize) + var props : VkPhysicalDeviceProperties + vkGetPhysicalDeviceProperties(phys, props) + v.max_storage_range = int64(props.limits.maxStorageBufferRange) + v.vram_mb = resolve_weight_cap(query_heap_info(phys)) / 1_000_000l +} + +// The DlimConfiguration vulkan source — RESOLVED values (the old tag folded raw env strings, +// so "auto" hid device-derived byte differences and a moving heap budget). +def private vk_dlim_source(var v : DlimVulkanConfig) : bool { + if (g_vk_dry_want) { + // an offline bake IS its config — echo it, never probe (there may be no device at all) + v = g_vk_dry_cfg + return true + } if (!vulkan_moe_gpu_arm()) { + return false + } + vk_dlim_wants_fill(v) + if (g_gpu != null) { + vk_caps_fill(g_gpu.phys, v) + } else { + var inscope instance <- create_instance("dasLLAMA dlim config", make_api_version(1u, 3u, 0u)) + volkLoadInstance(boost_value_to_vk(instance)) + vk_caps_fill(select_physical_device(instance), v) + } + return true +} + +def private vk_bake_tag : string { + var v : DlimVulkanConfig + if (!vk_dlim_source(v)) { return "" } - let cm = env_str("DASLLAMA_COOPMAT", "auto") - let vb = env_str("DASLLAMA_GPU_VRAM_MB", "auto") - return ("m{cm} b{vb}" - + " l{gpu_want_moe_layers()} s{gpu_want_moe_stream()}" - + " r{gpu_want_dn() ? 1 : 0}{gpu_want_attn() ? 1 : 0}{gpu_want_dnd() ? 1 : 0}{gpu_want_shexp() ? 1 : 0}{gpu_want_qkv() ? 1 : 0}{gpu_want_cls() ? 1 : 0}") + return dlim_vulkan_tag(v) } [init] @@ -8907,6 +9288,10 @@ def dasllama_math_vulkan_register() { needs_repack = false, priority = -1, available = @@vulkan_backend_available)) set_moe_gpu_arm("vulkan", @@vulkan_moe_gpu_arm) set_moe_gpu_bake_tag(@@vk_bake_tag) + set_dlim_vulkan_source(@@vk_dlim_source) + set_moe_gpu_dry_bake_arm(@@vulkan_dry_bake_arm) + set_moe_gpu_unmap_notify(@@vk_unmap_notify) + set_moe_gpu_would_accept_fn(@@vk_moe_would_accept) set_moe_gpu_vram_report(@@vk_resident_bytes) set_moe_gpu_device_report(@@vk_device_name) set_moe_gpu_budget_hooks(@@vk_weight_budget, @@vk_plane_bytes) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 8be5d63a12..97566a3f17 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -10,6 +10,7 @@ require dasllama/dasllama_env // env_int/env_flag: the shared typed readers + t require metal/das_metal_boost require dasllama/dasllama_common require dasllama/dasllama_math +require dasllama/dasllama_metal_shapes public // the extracted portable shape gates — re-exported so the driver family keeps its names // Shared infrastructure for the dasMetal resident drivers (the format-matrix arc's W1 split): // device/queue init, buffer pools, the weight-region + block_q8_0-blob + cat-blob caches, the @@ -34,111 +35,6 @@ var g_attn_single_max = 64l // the fused add+rms kernel's threadgroup row slab — wider models take the unfused pair let ADD_RMS_MAX_DIM = 4096l -// why a step declined to the CPU path — every gate reports its FIRST failing clause, and the -// declines-by-reason tables make silent coverage regressions observable (the support-matrix test -// asserts these names as strings; renaming a value is a test-visible change) -enum MetalDecodeDecline { - none // supported — the step runs on the GPU - quant_mode // weights not loaded under QuantMode.q8 - kquant_native // per-tensor K-quant planes: not readable as qblob offsets (kernels = W4) - arch // RETIRED (family waves A): the arch-name check is gone — non-std block - // sets decline `graph`, feature-flag configs decline `feature` per bit - backend_repack // repack backend: interleaved weight layout unreadable by the GPU kernels - rope_mode // rope tables off - config // RETIRED (family waves A): split into `feature` + the per-need - // `need_*` keys in the declines-by-reason table - graph // block set is not the std dense stack (MoE / recurrent / custom attn) - feature // the model needs layer-stack features this path's kernels don't - // implement — each missing bit records a `need_*` key alongside - shape // head/dim alignment outside the kernel grids - layers // per-layer non-uniformity (kv_dim/head_size/hidden/sliding/missing wv) - kv_dtype // session KV codec not uniform f16/f32 (the mirror moves raw row bytes) - depth // RETIRED (M2.4): the chunked/partD attention scans any depth — capacity - // limits surface as `mirror` (the arena cannot hold the session's slice) - no_uid // session without a real uid (mirror keying) - rope_rows // the position's rope row not built - batch_dim_cap // RETIRED (M2.6): the batch add+rms falls back to the unfused pair past - // the fused slab, and the GEMM's fixed K-tiles serve any dim at B > 4 - device // metal_decode_init failed (no device / PSO build failure) - mirror // the mirror arena cannot hold the KV slice - gpu_error // dispatch failed at execution time - planar // weights not in the blob-image form (the drivers are blob-only — - // load_model under metal mode transforms/maps the metal flavor) - dn_state // deltanet recurrent state not GPU-resident for this position (state is - // forward-only and NOT re-derivable — no watermark repair exists) -} - -//! Layer-stack features a model's graph needs beyond the base llama dense block — ONE derivation -//! (`metal_needs`) feeds every GPU gate; each path holds its own implemented-set mask and declines -//! `feature` on the difference, recording a `need_*` key per missing bit. -bitfield MetalNeed { - neox // NEOX rope pairing (j, j+hs/2) — qwen2/qwen3/phi3/gemma/gptoss - qkv_bias // additive bias on the Q/K/V projections — qwen2/glm4moe/gptoss - qk_norm // per-head RMSNorm on Q/K before rope — qwen3/gemma3+/qwen35 - v_norm // weightless per-head RMSNorm on V — gemma4 - partial_rope // rope_dim < head_size rotates only the first rope_dim dims — qwen35/glm4moe - q_gated // 2x-wide Q, out ⊙ σ(gate) before wo — qwen35 - attn_scale // attention-score scale override (not 1/sqrt(hs)) — gemma4 - softcap // attention-logit soft cap — gemma2 - sliding // sliding-window attention span — gemma2/3/4, gptoss - sinks // per-head sink logit in the softmax denominator — gptoss - out_bias // additive bias on the attention output projection — gptoss - pre_post_norm // extra post-attention / post-FFN norms — gemma2/3/4 - out_scale // per-layer scalar on the layer output — gemma4 - geglu // GeGLU FFN activation (gelu, not silu) — gemma2/3/4 - ple // per-layer embeddings — gemma4 E-series - qd_neq_dim // n_heads*head_size != dim (query width differs from the residual) — qwen3 -} - -//! The feature set model `t` needs from a resident GPU driver — derived from the loaded config -//! (per-file truth: loader-detected flags included). Graph-shape needs are NOT bits here — the -//! block set declares those (`attn_is_std` / `ffn_is_dense`) and the gates decline `graph` first. -def metal_needs(t : Model) : MetalNeed { - let c = t.config - var n : MetalNeed - n.neox = c.rope_neox - n.qkv_bias = c.attn_qkv_bias - n.qk_norm = c.qk_norm - n.v_norm = c.v_norm - n.partial_rope = c.rope_dim > 0l && c.rope_dim != c.head_size - n.q_gated = c.q_gated - n.attn_scale = c.attn_scale > 0.0 - n.softcap = c.attn_logit_softcap > 0.0 - n.sliding = c.sliding_window > 0l - n.sinks = c.attn_sinks - n.out_bias = c.attn_out_bias - n.pre_post_norm = c.pre_post_norm - n.out_scale = c.layer_out_scale - n.geglu = c.ffn_act != FfnAct.silu - n.ple = c.n_embd_per_layer > 0l - n.qd_neq_dim = c.n_heads * c.head_size != c.dim - return n -} - -def metal_missing_needs(t : Model; ok : MetalNeed) : MetalNeed { - return metal_needs(t) & ~ok -} - -// one `need_*` key per missing bit — the support-matrix cells assert these names as strings -def record_needs(var tab : table; missing : MetalNeed) { - if (missing.neox) { tab["need_neox"]++ } - if (missing.qkv_bias) { tab["need_qkv_bias"]++ } - if (missing.qk_norm) { tab["need_qk_norm"]++ } - if (missing.v_norm) { tab["need_v_norm"]++ } - if (missing.partial_rope) { tab["need_partial_rope"]++ } - if (missing.q_gated) { tab["need_q_gated"]++ } - if (missing.attn_scale) { tab["need_attn_scale"]++ } - if (missing.softcap) { tab["need_softcap"]++ } - if (missing.sliding) { tab["need_sliding"]++ } - if (missing.sinks) { tab["need_sinks"]++ } - if (missing.out_bias) { tab["need_out_bias"]++ } - if (missing.pre_post_norm) { tab["need_pre_post_norm"]++ } - if (missing.out_scale) { tab["need_out_scale"]++ } - if (missing.geglu) { tab["need_geglu"]++ } - if (missing.ple) { tab["need_ple"]++ } - if (missing.qd_neq_dim) { tab["need_qd_neq_dim"]++ } -} - var g_dev : MetalDevice? var g_queue : MetalCommandQueue? var g_failed = false @@ -702,35 +598,6 @@ def mx4_of(dev : MetalDevice?; t : Model; off : int64) : tuple) : bool { - for (f in a) { - if (!kq_fmt_gpu_supported(f)) { - return false - } - } - return true -} - -// every tensor fmt a kq load can route to a GPU kernel — the decode/batch/prefill gates all -// clause on this (the MoE expert planes never reach the llama-family GPU graph, so we1/2/3 -// stay out of scope) -def kq_load_gpu_supported(t : Model) : bool { - if (!t.kquant_native) { - return true - } - return ((kq_fmts_gpu_supported(t.wq_fmt) && kq_fmts_gpu_supported(t.wk_fmt) && - kq_fmts_gpu_supported(t.wv_fmt) && kq_fmts_gpu_supported(t.wo_fmt)) && - (kq_fmts_gpu_supported(t.w1_fmt) && kq_fmts_gpu_supported(t.w2_fmt) && - kq_fmts_gpu_supported(t.w3_fmt)) && - (kq_fmt_gpu_supported(t.wcls_fmt) && kq_fmt_gpu_supported(t.emb_fmt))) -} // ===== zero-copy logits: bytesNoCopy wrappers over session-owned storage ===== // Writes land straight into s.logits (no landing memcpy); borrowed, no deallocator. Gated on @@ -863,26 +730,6 @@ def mirror_evict_to_cap(cap_bytes : uint64; keep : uint64) { } } -//! Cross-layer KV sharing (gemma-4 E-series): does layer l either own its rows or share a -//! byte-compatible source? A shared layer aliases its source's mirror slab / prefill panel via -//! kv_row_prefix, so the source must own rows and match l's class geometry exactly. -def kv_share_ok(t : Model; l : int64) : bool { - if (l >= long_length(t.kv_src)) { // the MTP draft slab owns its rows - return true - } - let src = t.kv_src[l] - if (src == l) { - return true - } - return (src >= 0l && src < l && t.kv_src[src] == src && !layer_is_recurrent(t.config, src) && - layer_kv_dim(t.config, l) == layer_kv_dim(t.config, src) && - layer_head_size(t.config, l) == layer_head_size(t.config, src)) -} - -//! true when layer l reads another layer's KV (no rows of its own — Q-only attention) -def kv_shared_layer(t : Model; l : int64) : bool { - return l < long_length(t.kv_src) && t.kv_src[l] != l -} // Hetero-safe per-layer mirror addressing (gemma4's two kv-head classes): every KV codec is // linear bytes-per-element for kvd % 32 == 0, so the layout_offsets element prefix scales @@ -1100,13 +947,6 @@ def mirror_prepare(t : Model; var s : Session; pos : int64) : tuple= 2l && c.ssm_d_conv <= 8l -} // ===== deltanet recurrent-state mirrors (Wave D) — per-session GPU S + conv state ===== // Forward-only, NOT re-derivable: pos 0 zeroes, s.dn_pos == pos syncs from CPU, else decline; @@ -1239,110 +1079,7 @@ def private moe_fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt. // one MoE weight stack's (fmt, offset) serve check: base offset = the format's BIND alignment // (q8/k4/k5 %256, k6's d-plane pair %512); per-expert stride = the in-kernel index unit -// (q8 32-blocks, kq 256-superblocks — the expert shift is index math, no base multiple) -def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { - if (fmt == KqFmt.q8) { - return off >= 0l && (off % 256l) == 0l && (ege % 32l) == 0l - } - if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { - return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l - } - if (fmt == KqFmt.k6) { - return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l - } - if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned - return off >= 0l && (off % 128l) == 0l && (ege % 32l) == 0l - } - return false -} -// The MoE serve gate — covered graph shapes: plain routed FFN (qwen3moe, silu, no biases) -// + optional sigmoid-gated shared expert (qwen35moe), gemma4's dense-shared dual branch, and -// gpt-oss's mx4 experts (softmax_weight, biases, disk-order). No dense-lead / selection bias. -def moe_metal_ok(t : Model) : bool { - let c = t.config - if (!c.moe || c.n_layer_dense_lead > 0l || c.moe_exp_probs || - (c.expert_weights_scale != 0.0 && c.expert_weights_scale != 1.0)) { - return false - } - if (t.experts_mx4) { - // gpt-oss: softmax_weight top-k on raw logits, router + per-expert biases, clamped - // swiglu, disk-order mx4 stacks (the CPU laneq repack has no GPU carry) - if (c.moe_gate != MoeGate.softmax_weight || c.ffn_act != FfnAct.swiglu_oai || - !c.moe_router_bias || !c.moe_exps_bias || c.moe_dense_shexp || c.n_ff_shexp > 0l || - t.mx4_repacked || - t.router_b_off < 0l || t.web1_off < 0l || t.web2_off < 0l || t.web3_off < 0l) { - return false - } - } elif (c.moe_dense_shexp) { - if (c.moe_gate != MoeGate.softmax || !c.norm_topk_prob || - c.moe_router_bias || c.moe_exps_bias || c.n_ff_shexp > 0l || - c.ffn_act != FfnAct.gelu || t.moe_router_scale_off < 0l || t.moe_down_scale_off < 0l || - t.rms_pre_ffn2_off < 0l || t.rms_post_ffn1_off < 0l || t.rms_post_ffn2_off < 0l || - long_length(t.w1_offs) < c.n_layers) { - return false - } - } elif (c.moe_gate != MoeGate.softmax || - c.moe_router_bias || c.moe_exps_bias || c.ffn_act != FfnAct.silu) { - // norm_topk_prob both ways: renorm = select gmode 0 (qwen3moe/qwen35moe), no-renorm = - // gmode 2 (qwen2moe) - return false - } elif (c.n_ff_shexp > 0l) { - // sigmoid-gated shared expert (qwen35moe): q8 wsh planes at base + l*she ride the - // dense site kernels — base + stride %256 (bind), nsh %64 (prefill C tiles) - let she = c.dim * c.n_ff_shexp - if (!c.moe_shexp_gated || - (c.n_ff_shexp % 64l) != 0l || (she % 256l) != 0l || - (t.wsh1_off % 256l) != 0l || (t.wsh2_off % 256l) != 0l || (t.wsh3_off % 256l) != 0l) { - return false - } - } - let ne = c.n_expert - let k = c.n_expert_used - let nfe = c.n_ff_exp - let ege = c.dim * nfe - // select holds 8 logits per lane (ne <= 256); %64 dims = the gathered mul_mm C tiles - if (ne < 1l || ne > 256l || k < 1l || k > 16l || k > ne || nfe <= 0l || - (c.dim % 64l) != 0l || (nfe % 64l) != 0l || - ne * ege > 3800000000l || // uint32 in-kernel indices (q8 byte view = elements * 34/32) - t.router_off < 0l || empty(t.we1_offs) || empty(t.we2_offs) || empty(t.we3_offs)) { - return false - } - if (t.experts_mx4) { - // mx4 binds: nibble bytes at off/2 (%4 => off %8), E8M0 at off/32 (%4 => off %128) - if ((ege % 32l) != 0l) { - return false - } - for (l in range64(c.n_layers)) { - if (t.we1_offs[l] < 0l || (t.we1_offs[l] % 128l) != 0l || - t.we2_offs[l] < 0l || (t.we2_offs[l] % 128l) != 0l || - t.we3_offs[l] < 0l || (t.we3_offs[l] % 128l) != 0l) { - return false - } - } - return true - } - for (l in range64(c.n_layers)) { - let f1 = moe_fmt_at(t.we1_fmt, l) - let f2 = moe_fmt_at(t.we2_fmt, l) - let f3 = moe_fmt_at(t.we3_fmt, l) - // per-site alignment + reduction dims: superblock kq kernels iterate whole 256-rows, - // q51 whole 32-blocks (+ the mul_mm's 64-col output tiles) - let sb1 = f1 == KqFmt.k4 || f1 == KqFmt.k5 || f1 == KqFmt.k6 - let sb3 = f3 == KqFmt.k4 || f3 == KqFmt.k5 || f3 == KqFmt.k6 - let sb2 = f2 == KqFmt.k4 || f2 == KqFmt.k5 || f2 == KqFmt.k6 - if (!moe_site_ok(f1, t.we1_offs[l], ege) || - !moe_site_ok(f2, t.we2_offs[l], ege) || - !moe_site_ok(f3, t.we3_offs[l], ege) || - ((sb1 || sb3) && (c.dim % 256l) != 0l) || - (sb2 && (nfe % 256l) != 0l) || - ((f1 == KqFmt.q51 || f3 == KqFmt.q51) && ((c.dim % 32l) != 0l || (nfe % 64l) != 0l)) || - (f2 == KqFmt.q51 && ((nfe % 32l) != 0l || (c.dim % 64l) != 0l))) { - return false - } - } - return true -} // ===== the range tracker (the dispatch port; since P3 the batch rail's only hazard system) ===== diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_llama.das b/modules/dasLLAMA/dasllama/dasllama_metal_llama.das index ae9010e105..a595a2fdf3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_llama.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_llama.das @@ -54,64 +54,7 @@ def public metal_decode_shutdown { metal_common_release() } -// what each decode path's kernels implement beyond the base llama dense block — grows wave by wave. -let DECODE_NEEDS_OK = (MetalNeed.neox | MetalNeed.qkv_bias | MetalNeed.qk_norm | MetalNeed.qd_neq_dim | - MetalNeed.softcap | MetalNeed.sliding | MetalNeed.pre_post_norm | MetalNeed.geglu | - MetalNeed.v_norm | MetalNeed.attn_scale | MetalNeed.out_scale | - MetalNeed.sinks | MetalNeed.out_bias | MetalNeed.partial_rope | MetalNeed.q_gated | - MetalNeed.ple) -// q_gated/partial_rope only occur on hybrid models (attn_is_std false), which batch graph-declines; -// ple (E-series) has no batch arm either — batched steps decline `feature` and fall back per-row -let BATCH_NEEDS_OK = DECODE_NEEDS_OK & ~MetalNeed.ple // clear ple explicitly (XOR would re-add it if DECODE ever drops it) -def private decode_shape_decline(t : Model; needs_ok : MetalNeed; allow_moe : bool = false; allow_kv_share : bool = false) : MetalDecodeDecline { - if (t.quant != QuantMode.q8) { - return MetalDecodeDecline.quant_mode - } - if (!kq_load_gpu_supported(t)) { // a fmt without a GPU kernel (the q4_0 QAT tier, ...) must not dispatch - return MetalDecodeDecline.kquant_native - } - // non-std graphs decline; MoE (Wave C) + deltanet hybrid (Wave D) serve when their kernels cover it - if ((!t.blocks.attn_is_std && !(t.config.hybrid_deltanet && dn_metal_ok(t))) || - (!t.blocks.ffn_is_dense && !(allow_moe && moe_metal_ok(t)))) { - return MetalDecodeDecline.graph - } - if (kernel_backend_needs_repack(active_kernel_backend())) { - return MetalDecodeDecline.backend_repack - } - if (!get_rope_table_mode()) { - return MetalDecodeDecline.rope_mode - } - if (metal_missing_needs(t, needs_ok) != default) { - return MetalDecodeDecline.feature - } - let c = t.config - let hidden = layer_hidden(t, 0l) - // sliding masking lives on the CHUNKED path only — window >= 128 (the single-form depth ceiling's clamp max) guarantees the single form never sees a masked row; decline tighter - if (((c.dim % 32l) != 0l || (hidden % 32l) != 0l) || - (c.sliding_window > 0l && c.sliding_window < 128l)) { - return MetalDecodeDecline.shape - } - // per-CLASS: whole simdgroups; red[16] + the epl <= 16 q-register hoist cap the threadgroup width at 512 (gemma4's global class) - for (l in range64(c.n_layers)) { - if (layer_is_recurrent(c, l)) { // no KV, no attention tensors — only the FFN uniformity applies - if (layer_hidden(t, l) != hidden) { - return MetalDecodeDecline.layers - } - continue - } - let hs_l = layer_head_size(c, l) - if ((hs_l % 32l) != 0l || hs_l > 512l || (layer_kv_dim(c, l) % 32l) != 0l) { - return MetalDecodeDecline.shape - } - // E-series KV sharing: only the single-stream driver has the Q-only arm (allow_kv_share) - if ((t.kv_src[l] != l && !(allow_kv_share && kv_share_ok(t, l))) || layer_hidden(t, l) != hidden) { - return MetalDecodeDecline.layers - } - // a missing attn_v (V = the pre-norm K projection) serves — raw copy without v_norm; both encoded (the CPU mm_qkv shapes) - } - return MetalDecodeDecline.none -} // ... plus the decode-specific gates (uniform f16/f32 KV, attention depth, a real session uid) [unused_argument(pos)] // depth retired (M2.4) — pos stays in the gate signature for future clauses @@ -147,17 +90,6 @@ def private decode_decline(t : Model; s : Session; pos : int64) : MetalDecodeDec def private kq_fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt.q8 : a[l] -// mint-time servability (register_metal_servable): shape/graph/feature gates only — the planar -// check is a RUNTIME gate. Tied-fp32 classifiers decline: the blob drivers force GPU logits -// and have no fp32-table GEMV. -def private metal_model_servable_impl(t : Model) : bool { - let cls_fmt = t.config.shared_weights ? t.emb_fmt : t.wcls_fmt - // tied-fp32 classifier declines (no fp32-table GEMV); NextN serves — prefill stashes mtp_h + warms the slab - if (t.config.shared_weights && !t.cls_q8 && cls_fmt == KqFmt.q8) { - return false - } - return decode_shape_decline(t, DECODE_NEEDS_OK, true, true) == MetalDecodeDecline.none -} // Can the speculative chain feed itself on this model? It needs a GPU embed gather for the // argmax winner: the q8 linear copy (cls_q8), or a tied k6 classifier (every K-quant file on @@ -3306,7 +3238,6 @@ def dasllama_metal_llama_register() { register_batch_decode_override("metal", @@metal_batch_decode_forward) register_mtp_spec_override("metal", @@metal_mtp_spec_eval) register_mtp_seam_override("metal", @@metal_mtp_seam_row) - register_metal_servable(@@metal_model_servable_impl) // load_model's image-flavor gate g_env_logits = env_flag("DASLLAMA_METAL_LOGITS", true) g_env_batch_mm = env_flag("DASLLAMA_METAL_BATCH_MM", false) g_env_attn_d = env_flag("DASLLAMA_METAL_ATTN_D", true) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das new file mode 100644 index 0000000000..080be89113 --- /dev/null +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -0,0 +1,361 @@ +options gen2 +options _comment_hygiene = true + +module dasllama_metal_shapes shared public + +require dasllama/dasllama_common +require dasllama/dasllama_math // kernel_backend_needs_repack / active_kernel_backend — the repack clause + +// The Metal drivers' MODEL-SHAPE gates, extracted PORTABLE (no das_metal require): the decline +// enum, the feature-bit derivation, and every pure-Model serve gate the mint-time servable +// predicate needs. This is what lets a PC bake an M1's metal-flavor .dlim — the blob transform +// (dasllama_layout) was always pure bytes; only these gates were build-locked to Apple. +// The RUNTIME gates (device, PSO, planar, mirrors) stay in the metal modules. + +// why a step declined to the CPU path — every gate reports its FIRST failing clause, and the +// declines-by-reason tables make silent coverage regressions observable (the support-matrix test +// asserts these names as strings; renaming a value is a test-visible change) +enum MetalDecodeDecline { + none // supported — the step runs on the GPU + quant_mode // weights not loaded under QuantMode.q8 + kquant_native // per-tensor K-quant planes: not readable as qblob offsets (kernels = W4) + arch // RETIRED (family waves A): the arch-name check is gone — non-std block + // sets decline `graph`, feature-flag configs decline `feature` per bit + backend_repack // repack backend: interleaved weight layout unreadable by the GPU kernels + rope_mode // rope tables off + config // RETIRED (family waves A): split into `feature` + the per-need + // `need_*` keys in the declines-by-reason table + graph // block set is not the std dense stack (MoE / recurrent / custom attn) + feature // the model needs layer-stack features this path's kernels don't + // implement — each missing bit records a `need_*` key alongside + shape // head/dim alignment outside the kernel grids + layers // per-layer non-uniformity (kv_dim/head_size/hidden/sliding/missing wv) + kv_dtype // session KV codec not uniform f16/f32 (the mirror moves raw row bytes) + depth // RETIRED (M2.4): the chunked/partD attention scans any depth — capacity + // limits surface as `mirror` (the arena cannot hold the session's slice) + no_uid // session without a real uid (mirror keying) + rope_rows // the position's rope row not built + batch_dim_cap // RETIRED (M2.6): the batch add+rms falls back to the unfused pair past + // the fused slab, and the GEMM's fixed K-tiles serve any dim at B > 4 + device // metal_decode_init failed (no device / PSO build failure) + mirror // the mirror arena cannot hold the KV slice + gpu_error // dispatch failed at execution time + planar // weights not in the blob-image form (the drivers are blob-only — + // load_model under metal mode transforms/maps the metal flavor) + dn_state // deltanet recurrent state not GPU-resident for this position (state is + // forward-only and NOT re-derivable — no watermark repair exists) +} + +//! Layer-stack features a model's graph needs beyond the base llama dense block — ONE derivation +//! (`metal_needs`) feeds every GPU gate; each path holds its own implemented-set mask and declines +//! `feature` on the difference, recording a `need_*` key per missing bit. +bitfield MetalNeed { + neox // NEOX rope pairing (j, j+hs/2) — qwen2/qwen3/phi3/gemma/gptoss + qkv_bias // additive bias on the Q/K/V projections — qwen2/glm4moe/gptoss + qk_norm // per-head RMSNorm on Q/K before rope — qwen3/gemma3+/qwen35 + v_norm // weightless per-head RMSNorm on V — gemma4 + partial_rope // rope_dim < head_size rotates only the first rope_dim dims — qwen35/glm4moe + q_gated // 2x-wide Q, out ⊙ σ(gate) before wo — qwen35 + attn_scale // attention-score scale override (not 1/sqrt(hs)) — gemma4 + softcap // attention-logit soft cap — gemma2 + sliding // sliding-window attention span — gemma2/3/4, gptoss + sinks // per-head sink logit in the softmax denominator — gptoss + out_bias // additive bias on the attention output projection — gptoss + pre_post_norm // extra post-attention / post-FFN norms — gemma2/3/4 + out_scale // per-layer scalar on the layer output — gemma4 + geglu // GeGLU FFN activation (gelu, not silu) — gemma2/3/4 + ple // per-layer embeddings — gemma4 E-series + qd_neq_dim // n_heads*head_size != dim (query width differs from the residual) — qwen3 +} + +//! The feature set model `t` needs from a resident GPU driver — derived from the loaded config +//! (per-file truth: loader-detected flags included). Graph-shape needs are NOT bits here — the +//! block set declares those (`attn_is_std` / `ffn_is_dense`) and the gates decline `graph` first. +def metal_needs(t : Model) : MetalNeed { + let c = t.config + var n : MetalNeed + n.neox = c.rope_neox + n.qkv_bias = c.attn_qkv_bias + n.qk_norm = c.qk_norm + n.v_norm = c.v_norm + n.partial_rope = c.rope_dim > 0l && c.rope_dim != c.head_size + n.q_gated = c.q_gated + n.attn_scale = c.attn_scale > 0.0 + n.softcap = c.attn_logit_softcap > 0.0 + n.sliding = c.sliding_window > 0l + n.sinks = c.attn_sinks + n.out_bias = c.attn_out_bias + n.pre_post_norm = c.pre_post_norm + n.out_scale = c.layer_out_scale + n.geglu = c.ffn_act != FfnAct.silu + n.ple = c.n_embd_per_layer > 0l + n.qd_neq_dim = c.n_heads * c.head_size != c.dim + return n +} + +def metal_missing_needs(t : Model; ok : MetalNeed) : MetalNeed { + return metal_needs(t) & ~ok +} + +// one `need_*` key per missing bit — the support-matrix cells assert these names as strings +def record_needs(var tab : table; missing : MetalNeed) { + if (missing.neox) { tab["need_neox"]++ } + if (missing.qkv_bias) { tab["need_qkv_bias"]++ } + if (missing.qk_norm) { tab["need_qk_norm"]++ } + if (missing.v_norm) { tab["need_v_norm"]++ } + if (missing.partial_rope) { tab["need_partial_rope"]++ } + if (missing.q_gated) { tab["need_q_gated"]++ } + if (missing.attn_scale) { tab["need_attn_scale"]++ } + if (missing.softcap) { tab["need_softcap"]++ } + if (missing.sliding) { tab["need_sliding"]++ } + if (missing.sinks) { tab["need_sinks"]++ } + if (missing.out_bias) { tab["need_out_bias"]++ } + if (missing.pre_post_norm) { tab["need_pre_post_norm"]++ } + if (missing.out_scale) { tab["need_out_scale"]++ } + if (missing.geglu) { tab["need_geglu"]++ } + if (missing.ple) { tab["need_ple"]++ } + if (missing.qd_neq_dim) { tab["need_qd_neq_dim"]++ } +} + +// the kq formats the Metal kernels implement — anything else (the q4_0 QAT tier, future +// formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch +// (the dispatchers below treat "not k4/k6" as k5) +def kq_fmt_gpu_supported(f : KqFmt) : bool { + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 +} + +def private kq_fmts_gpu_supported(a : array) : bool { + for (f in a) { + if (!kq_fmt_gpu_supported(f)) { + return false + } + } + return true +} + +// every tensor fmt a kq load can route to a GPU kernel — the decode/batch/prefill gates all +// clause on this (the MoE expert planes never reach the llama-family GPU graph, so we1/2/3 +// stay out of scope) +def kq_load_gpu_supported(t : Model) : bool { + if (!t.kquant_native) { + return true + } + return ((kq_fmts_gpu_supported(t.wq_fmt) && kq_fmts_gpu_supported(t.wk_fmt) && + kq_fmts_gpu_supported(t.wv_fmt) && kq_fmts_gpu_supported(t.wo_fmt)) && + (kq_fmts_gpu_supported(t.w1_fmt) && kq_fmts_gpu_supported(t.w2_fmt) && + kq_fmts_gpu_supported(t.w3_fmt)) && + (kq_fmt_gpu_supported(t.wcls_fmt) && kq_fmt_gpu_supported(t.emb_fmt))) +} + +//! Cross-layer KV sharing (gemma-4 E-series): does layer l either own its rows or share a +//! byte-compatible source? A shared layer aliases its source's mirror slab / prefill panel via +//! kv_row_prefix, so the source must own rows and match l's class geometry exactly. +def kv_share_ok(t : Model; l : int64) : bool { + if (l >= long_length(t.kv_src)) { // the MTP draft slab owns its rows + return true + } + let src = t.kv_src[l] + if (src == l) { + return true + } + return (src >= 0l && src < l && t.kv_src[src] == src && !layer_is_recurrent(t.config, src) && + layer_kv_dim(t.config, l) == layer_kv_dim(t.config, src) && + layer_head_size(t.config, l) == layer_head_size(t.config, src)) +} + +//! true when layer l reads another layer's KV (no rows of its own — Q-only attention) +def kv_shared_layer(t : Model; l : int64) : bool { + return l < long_length(t.kv_src) && t.kv_src[l] != l +} + +// Wave D serve gate for the deltanet kernel set: fixed d_state 128 (the scan/l2/gate lane math +// bakes 32 lanes x 4), conv taps <= 8 (hist registers). β/α projections: q8 planes ride the +// site GEMVs; f32-on-disk (dn_ba_f32) rides the router-slab f32 GEMV off the fblob offsets. +def dn_metal_ok(t : Model) : bool { + let c = t.config + return c.ssm_d_state == 128l && c.ssm_d_conv >= 2l && c.ssm_d_conv <= 8l +} + +// (q8 32-blocks, kq 256-superblocks — the expert shift is index math, no base multiple) +def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { + if (fmt == KqFmt.q8) { + return off >= 0l && (off % 256l) == 0l && (ege % 32l) == 0l + } + if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { + return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l + } + if (fmt == KqFmt.k6) { + return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l + } + if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned + return off >= 0l && (off % 128l) == 0l && (ege % 32l) == 0l + } + return false +} + +def private moe_fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt.q8 : a[l] + +// The MoE serve gate — covered graph shapes: plain routed FFN (qwen3moe, silu, no biases) +// + optional sigmoid-gated shared expert (qwen35moe), gemma4's dense-shared dual branch, and +// gpt-oss's mx4 experts (softmax_weight, biases, disk-order). No dense-lead / selection bias. +def moe_metal_ok(t : Model) : bool { + let c = t.config + if (!c.moe || c.n_layer_dense_lead > 0l || c.moe_exp_probs || + (c.expert_weights_scale != 0.0 && c.expert_weights_scale != 1.0)) { + return false + } + if (t.experts_mx4) { + // gpt-oss: softmax_weight top-k on raw logits, router + per-expert biases, clamped + // swiglu, disk-order mx4 stacks (the CPU laneq repack has no GPU carry) + if (c.moe_gate != MoeGate.softmax_weight || c.ffn_act != FfnAct.swiglu_oai || + !c.moe_router_bias || !c.moe_exps_bias || c.moe_dense_shexp || c.n_ff_shexp > 0l || + t.mx4_repacked || + t.router_b_off < 0l || t.web1_off < 0l || t.web2_off < 0l || t.web3_off < 0l) { + return false + } + } elif (c.moe_dense_shexp) { + if (c.moe_gate != MoeGate.softmax || !c.norm_topk_prob || + c.moe_router_bias || c.moe_exps_bias || c.n_ff_shexp > 0l || + c.ffn_act != FfnAct.gelu || t.moe_router_scale_off < 0l || t.moe_down_scale_off < 0l || + t.rms_pre_ffn2_off < 0l || t.rms_post_ffn1_off < 0l || t.rms_post_ffn2_off < 0l || + long_length(t.w1_offs) < c.n_layers) { + return false + } + } elif (c.moe_gate != MoeGate.softmax || + c.moe_router_bias || c.moe_exps_bias || c.ffn_act != FfnAct.silu) { + // norm_topk_prob both ways: renorm = select gmode 0 (qwen3moe/qwen35moe), no-renorm = + // gmode 2 (qwen2moe) + return false + } elif (c.n_ff_shexp > 0l) { + // sigmoid-gated shared expert (qwen35moe): q8 wsh planes at base + l*she ride the + // dense site kernels — base + stride %256 (bind), nsh %64 (prefill C tiles) + let she = c.dim * c.n_ff_shexp + if (!c.moe_shexp_gated || + (c.n_ff_shexp % 64l) != 0l || (she % 256l) != 0l || + (t.wsh1_off % 256l) != 0l || (t.wsh2_off % 256l) != 0l || (t.wsh3_off % 256l) != 0l) { + return false + } + } + let ne = c.n_expert + let k = c.n_expert_used + let nfe = c.n_ff_exp + let ege = c.dim * nfe + // select holds 8 logits per lane (ne <= 256); %64 dims = the gathered mul_mm C tiles + if (ne < 1l || ne > 256l || k < 1l || k > 16l || k > ne || nfe <= 0l || + (c.dim % 64l) != 0l || (nfe % 64l) != 0l || + ne * ege > 3800000000l || // uint32 in-kernel indices (q8 byte view = elements * 34/32) + t.router_off < 0l || empty(t.we1_offs) || empty(t.we2_offs) || empty(t.we3_offs)) { + return false + } + if (t.experts_mx4) { + // mx4 binds: nibble bytes at off/2 (%4 => off %8), E8M0 at off/32 (%4 => off %128) + if ((ege % 32l) != 0l) { + return false + } + for (l in range64(c.n_layers)) { + if (t.we1_offs[l] < 0l || (t.we1_offs[l] % 128l) != 0l || + t.we2_offs[l] < 0l || (t.we2_offs[l] % 128l) != 0l || + t.we3_offs[l] < 0l || (t.we3_offs[l] % 128l) != 0l) { + return false + } + } + return true + } + for (l in range64(c.n_layers)) { + let f1 = moe_fmt_at(t.we1_fmt, l) + let f2 = moe_fmt_at(t.we2_fmt, l) + let f3 = moe_fmt_at(t.we3_fmt, l) + // per-site alignment + reduction dims: superblock kq kernels iterate whole 256-rows, + // q51 whole 32-blocks (+ the mul_mm's 64-col output tiles) + let sb1 = f1 == KqFmt.k4 || f1 == KqFmt.k5 || f1 == KqFmt.k6 + let sb3 = f3 == KqFmt.k4 || f3 == KqFmt.k5 || f3 == KqFmt.k6 + let sb2 = f2 == KqFmt.k4 || f2 == KqFmt.k5 || f2 == KqFmt.k6 + if (!moe_site_ok(f1, t.we1_offs[l], ege) || + !moe_site_ok(f2, t.we2_offs[l], ege) || + !moe_site_ok(f3, t.we3_offs[l], ege) || + ((sb1 || sb3) && (c.dim % 256l) != 0l) || + (sb2 && (nfe % 256l) != 0l) || + ((f1 == KqFmt.q51 || f3 == KqFmt.q51) && ((c.dim % 32l) != 0l || (nfe % 64l) != 0l)) || + (f2 == KqFmt.q51 && ((nfe % 32l) != 0l || (c.dim % 64l) != 0l))) { + return false + } + } + return true +} + +// what each decode path's kernels implement beyond the base llama dense block — grows wave by wave. +let DECODE_NEEDS_OK = (MetalNeed.neox | MetalNeed.qkv_bias | MetalNeed.qk_norm | MetalNeed.qd_neq_dim | + MetalNeed.softcap | MetalNeed.sliding | MetalNeed.pre_post_norm | MetalNeed.geglu | + MetalNeed.v_norm | MetalNeed.attn_scale | MetalNeed.out_scale | + MetalNeed.sinks | MetalNeed.out_bias | MetalNeed.partial_rope | MetalNeed.q_gated | + MetalNeed.ple) +// q_gated/partial_rope only occur on hybrid models (attn_is_std false), which batch graph-declines; +// ple (E-series) has no batch arm either — batched steps decline `feature` and fall back per-row +let BATCH_NEEDS_OK = DECODE_NEEDS_OK & ~MetalNeed.ple // clear ple explicitly (XOR would re-add it if DECODE ever drops it) + +def decode_shape_decline(t : Model; needs_ok : MetalNeed; allow_moe : bool = false; allow_kv_share : bool = false) : MetalDecodeDecline { + if (t.quant != QuantMode.q8) { + return MetalDecodeDecline.quant_mode + } + if (!kq_load_gpu_supported(t)) { // a fmt without a GPU kernel (the q4_0 QAT tier, ...) must not dispatch + return MetalDecodeDecline.kquant_native + } + // non-std graphs decline; MoE (Wave C) + deltanet hybrid (Wave D) serve when their kernels cover it + if ((!t.blocks.attn_is_std && !(t.config.hybrid_deltanet && dn_metal_ok(t))) || + (!t.blocks.ffn_is_dense && !(allow_moe && moe_metal_ok(t)))) { + return MetalDecodeDecline.graph + } + if (kernel_backend_needs_repack(active_kernel_backend())) { + return MetalDecodeDecline.backend_repack + } + if (!get_rope_table_mode()) { + return MetalDecodeDecline.rope_mode + } + if (metal_missing_needs(t, needs_ok) != default) { + return MetalDecodeDecline.feature + } + let c = t.config + let hidden = layer_hidden(t, 0l) + // sliding masking lives on the CHUNKED path only — window >= 128 (the single-form depth ceiling's clamp max) guarantees the single form never sees a masked row; decline tighter + if (((c.dim % 32l) != 0l || (hidden % 32l) != 0l) || + (c.sliding_window > 0l && c.sliding_window < 128l)) { + return MetalDecodeDecline.shape + } + // per-CLASS: whole simdgroups; red[16] + the epl <= 16 q-register hoist cap the threadgroup width at 512 (gemma4's global class) + for (l in range64(c.n_layers)) { + if (layer_is_recurrent(c, l)) { // no KV, no attention tensors — only the FFN uniformity applies + if (layer_hidden(t, l) != hidden) { + return MetalDecodeDecline.layers + } + continue + } + let hs_l = layer_head_size(c, l) + if ((hs_l % 32l) != 0l || hs_l > 512l || (layer_kv_dim(c, l) % 32l) != 0l) { + return MetalDecodeDecline.shape + } + // E-series KV sharing: only the single-stream driver has the Q-only arm (allow_kv_share) + if ((t.kv_src[l] != l && !(allow_kv_share && kv_share_ok(t, l))) || layer_hidden(t, l) != hidden) { + return MetalDecodeDecline.layers + } + // a missing attn_v (V = the pre-norm K projection) serves — raw copy without v_norm; both encoded (the CPU mm_qkv shapes) + } + return MetalDecodeDecline.none +} + +// mint-time servability (register_metal_servable): shape/graph/feature gates only — the planar +// check is a RUNTIME gate. Tied-fp32 classifiers decline: the blob drivers force GPU logits +// and have no fp32-table GEMV. +def private metal_model_servable_impl(t : Model) : bool { + let cls_fmt = t.config.shared_weights ? t.emb_fmt : t.wcls_fmt + // tied-fp32 classifier declines (no fp32-table GEMV); NextN serves — prefill stashes mtp_h + warms the slab + if (t.config.shared_weights && !t.cls_q8 && cls_fmt == KqFmt.q8) { + return false + } + return decode_shape_decline(t, DECODE_NEEDS_OK, true, true) == MetalDecodeDecline.none +} + +[init] +def private register_shape_gates { + // pure model shape, registered on EVERY build — a non-Apple converter can bake the metal flavor + register_metal_servable(@@metal_model_servable_impl) +} diff --git a/modules/dasLLAMA/dasllama/dasllama_transformer.das b/modules/dasLLAMA/dasllama/dasllama_transformer.das index c7947aa4a7..3346258e5d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_transformer.das +++ b/modules/dasLLAMA/dasllama/dasllama_transformer.das @@ -21,5 +21,6 @@ require dasllama/dasllama_arch_qwen35 // nolint:STYLE030 — side-effect requir require dasllama/dasllama_arch_gptoss // nolint:STYLE030 — side-effect require: fires the arch's [init] registration require dasllama/dasllama_arch_mistral3 // nolint:STYLE030 — side-effect require: fires the arch's [init] registration require dasllama/dasllama_arch_glm4moe // nolint:STYLE030 — side-effect require: fires the arch's [init] registration +require dasllama/dasllama_metal_shapes // nolint:STYLE030 — [init]-only: registers the PORTABLE metal servable predicate (pure model shape) — every build, so a non-Apple converter can bake the metal flavor require ?das_metal dasllama/dasllama_metal_prefill // nolint:STYLE030 — [init]-only: registers prefill override "metal" (the dasMetal GPU-resident path). STRICT guard: Apple builds only; it lives here (not dasllama_common) because it requires the engine back — the umbrella breaks the cycle require ?das_metal dasllama/dasllama_metal_llama // nolint:STYLE030 — [init]-only: registers decode override "metal" (same contract as the prefill line above) diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index b051f7af42..e085ec393a 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -62,6 +62,10 @@ carrier, model-free, runs in CI) `smol metal tower whisper voxtral`; the voxtral arm mints/maps the blob-only metal flavor (SmolLM2) incl. the CPU-tripwire and a teacher-forced logits-tolerance parity cell (greedy token equality is NOT a valid bar on a 135M — genuine near-ties flip on ~0.02 gaps under ~0.75 cross-backend noise). +The `image-vulkan` suite (test_model_image_vulkan, arm `vulkan`) covers the OFFLINE vulkan +bake: the runner arms DASLLAMA_GPU + a small VRAM budget so the probed config carries a +vulkan section, the DRY tier collects a role-stamped plan with no device calls (safe on +GPU-less boxes), and the flavor image round-trips the plan verbatim. ## Blob-only Metal fixtures (the two-model pattern) diff --git a/modules/dasLLAMA/tests/run.das b/modules/dasLLAMA/tests/run.das index c0f45570ba..593e3c6106 100644 --- a/modules/dasLLAMA/tests/run.das +++ b/modules/dasLLAMA/tests/run.das @@ -34,8 +34,8 @@ struct RunArgs { @clarg_doc = "Repeat count — explicit only (default 1, never best-of-N)" nreps : int = 1 - @clarg_doc = "daslang binary (default ./bin/daslang — run from the repo root)" - das : string = "./bin/daslang" + @clarg_doc = "daslang binary (default: the binary running this script — a hardcoded ./bin/daslang picked up a STALE top-level exe on the MSVC layout)" + das : string = "" @clarg_short = "?" @clarg_name = "show-help" @@ -54,6 +54,8 @@ def private suite_files(name : string) : array { return <- [ "modules/dasLLAMA/tests/test_metal_prefill_kernels.das" ] } elif (name == "image") { return <- [ "modules/dasLLAMA/tests/test_model_image.das" ] + } elif (name == "image-vulkan") { + return <- [ "modules/dasLLAMA/tests/test_model_image_vulkan.das" ] } elif (name == "all") { return <- [ "modules/dasLLAMA/tests/test_metal_decode_parity.das", "modules/dasLLAMA/tests/test_metal_prefill_parity.das", @@ -114,26 +116,50 @@ def main() : int { } var files <- suite_files(cfg.suite) if (empty(files)) { - to_log(LOG_ERROR, "unknown --suite '{cfg.suite}' (decode | prefill | matrix | kernels | image | all)\n") + to_log(LOG_ERROR, "unknown --suite '{cfg.suite}' (decode | prefill | matrix | kernels | image | image-vulkan | all)\n") return 2 } - let tmp = empty(get_env_variable("TMPDIR")) ? "/tmp" : get_env_variable("TMPDIR") + var tmp = get_env_variable("TMPDIR") + if (empty(tmp)) { + tmp = get_env_variable("TEMP") // the Windows spelling; TMPDIR is the POSIX one + } + if (empty(tmp)) { + tmp = "/tmp" + } // CPU_PREFILL: the suites' CPU truth halves and parity controls prefill on the CPU by intent. // HAZARD_STRICT: the scheduling-unification ratchet's runtime half — an undeclared dispatch // on an armed hazard oracle PANICS here, so declaration gaps can never ship quietly. - var envs = (cfg.full ? "DASLLAMA_PARITY_FULL=1" : "DASLLAMA_TEST_ARMS='{cfg.arm}'") + " DASLLAMA_CPU_PREFILL=1 DASLLAMA_METAL_HAZARD_STRICT=1" - if (!empty(cfg.family)) { // composes with either mode: a family-scoped full gate is the profiling-cadence default - envs = "{envs} DASLLAMA_TEST_FAMILY='{cfg.family}'" + // cmd.exe has no VAR=val prefix — `set "VAR=val"&&` is the safe form (llvm_tune's tuner_cmd) + let win = get_platform_name() == "windows" + var envs = "" + if (win) { + envs = ((cfg.full ? "set \"DASLLAMA_PARITY_FULL=1\"&& " : "set \"DASLLAMA_TEST_ARMS={cfg.arm}\"&& ") + + "set \"DASLLAMA_CPU_PREFILL=1\"&& set \"DASLLAMA_METAL_HAZARD_STRICT=1\"&& ") + if (!empty(cfg.family)) { // composes with either mode: a family-scoped full gate is the profiling-cadence default + envs = "{envs}set \"DASLLAMA_TEST_FAMILY={cfg.family}\"&& " + } + } else { + envs = (cfg.full ? "DASLLAMA_PARITY_FULL=1" : "DASLLAMA_TEST_ARMS='{cfg.arm}'") + " DASLLAMA_CPU_PREFILL=1 DASLLAMA_METAL_HAZARD_STRICT=1 " + if (!empty(cfg.family)) { + envs = "DASLLAMA_TEST_FAMILY='{cfg.family}' {envs}" + } + } + if (cfg.suite == "image-vulkan") { + // the offline-bake suite: arm the tier so the probed config carries a vulkan section — + // the bake itself is DRY (no device calls), so this is safe on GPU-less boxes too + envs = (win ? "{envs}set \"DASLLAMA_GPU=1\"&& set \"DASLLAMA_GPU_VRAM_MB=2000\"&& " + : "DASLLAMA_GPU=1 DASLLAMA_GPU_VRAM_MB=2000 {envs}") } // the full gate holds the 8B/70B tails — the stock 1200s dastest wall is an arm-mode budget let tmo = cfg.full ? 3600 : 1200 let reps = max(cfg.nreps, 1) + let das_exe = empty(cfg.das) ? get_command_line_arguments()[0] : cfg.das var failures = 0 for (rep in range(reps)) { for (file in files) { let tag = base_name(file) let log_path = path_join(tmp, "dasllama_run_{tag}_{rep}_{ref_time_ticks()}.log") - let cmd = "{envs} \"{cfg.das}\" -jit dastest/dastest.das -- --test \"{file}\" --timeout {tmo} > \"{log_path}\" 2>&1" + let cmd = "{envs}\"{das_exe}\" -jit dastest/dastest.das -- --test \"{file}\" --timeout {tmo} > \"{log_path}\" 2>&1" let rep_tag = reps > 1 ? " rep {rep + 1}/{reps}" : "" to_log(LOG_INFO, "START {tag}{rep_tag}: {cmd}\n") let t0 = ref_time_ticks() diff --git a/modules/dasLLAMA/tests/test_env_registry.das b/modules/dasLLAMA/tests/test_env_registry.das index 99a2768135..bed2bd1727 100644 --- a/modules/dasLLAMA/tests/test_env_registry.das +++ b/modules/dasLLAMA/tests/test_env_registry.das @@ -17,6 +17,7 @@ require daslib/strings_boost let SCAN_ROOTS = [ "modules/dasLLAMA", "utils/dasllama-server", + "utils/dasllama-convert", "examples/dasLLAMA", "tutorials/dasLLAMA" ] diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index 632d6c7bdd..2c764ea607 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -7,6 +7,7 @@ require dasllama/dasllama // the facade: load_model's image cache is require dasllama/dasllama_whisper require dasllama/dasllama_gguf // rd_u32/rd_u16 — the wav fixture reader require dasllama/dasllama_image +require dasllama/dasllama_config // DlimConfiguration — the peek/round-trip mechanics arms require dasllama/dasllama_math // pin_kernel_backend — the metal-flavor arm's donor contract require daslib/jobque_boost require daslib/archive @@ -214,6 +215,16 @@ def test_image_mechanics(t : T?) { t |> success(load_image(path, m2b, "img-probe"), "re-saved image maps") delete m2b + // the v7 meta head: peek reads version + identity + embedded config WITHOUT loading, + // and the embedded config re-derives the exact identity the image was baked under + let pk = image_peek(path) + t |> success(pk.valid, "peek: header sane") + t |> equal(pk.version, IMAGE_VERSION, "peek: version") + t |> equal(pk.identity, image_identity("img-probe"), "peek: baked identity string") + var pcfg = DlimConfiguration() + t |> success(dlim_config_from_json(pk.config_json, pcfg), "peek: embedded config JSON parses") + t |> equal(dlim_identity(pcfg, IMAGE_VERSION, "img-probe"), pk.identity, "peek: config re-derives the identity") + var m3 = ImgProbe() t |> success(!load_image(path, m3, "wrong-tag"), "identity mismatch declines") patch_u32(t, path, 4l, 0xDEADu) @@ -262,6 +273,19 @@ def test_image_mechanics(t : T?) { t |> success(caught, "forget on an owned locked array is a loud error") delete owned } + t |> run("DlimConfiguration JSON round-trips with the identity intact") @(t : T?) { + if (!arm_on(t, "mechanics")) { + return + } + let dc = dlim_config_current("q8") + let js = dlim_config_json(dc) + t |> success(!empty(js), "config serializes") + var back = DlimConfiguration() + t |> success(dlim_config_from_json(js, back), "JSON parses back") + t |> equal(dlim_identity(back, IMAGE_VERSION), dlim_identity(dc, IMAGE_VERSION), "identity survives the round trip") + var junk = DlimConfiguration() + t |> success(!dlim_config_from_json("not a config", junk), "garbage declines") + } } // ===== real carriers: gguf load vs mapped image load ===== diff --git a/modules/dasLLAMA/tests/test_model_image_vulkan.das b/modules/dasLLAMA/tests/test_model_image_vulkan.das new file mode 100644 index 0000000000..b4b88369a9 --- /dev/null +++ b/modules/dasLLAMA/tests/test_model_image_vulkan.das @@ -0,0 +1,79 @@ +options gen2 +options persistent_heap +options stack = 131072 + +require dastest/testing_boost public +// STYLE030 nolints: the whole test body is static_if'd on builtin_module_exists(vulkan), so a +// no-vulkan build compiles it away and every body-only require reads as unused +require dasllama/dasllama // nolint:STYLE029,STYLE030 — umbrella fires each arch [init] registration +require dasllama/dasllama_image // nolint:STYLE030 +require dasllama/dasllama_config // nolint:STYLE030 +require dasllama/dasllama_math +require daslib/jobque_boost +require daslib/fio // nolint:STYLE030 +require _model_tier + +// The OFFLINE vulkan bake (plan-as-data): the dry tier runs the load walk's accept/refuse +// arithmetic against the config's numbers with NO device — the converter's `-f vulkan` rail. +// Drive through run.das --suite image-vulkan (it arms DASLLAMA_GPU + a small VRAM budget); +// unarmed contexts skip cleanly, so the file is safe in any suite. + +[test] +def test_vulkan_dry_bake(tst : T?) { + tst |> run("SmolLM2 q8: dry bake collects a role-stamped plan and the image round-trips") @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!family_on(t, "llama")) return + if (!arm_on(t, "vulkan")) { + return + } + if (!jit_enabled()) { + t |> skip("interpreted (dasLLAMA model tests are JIT-only)") + return + } + let path = path_join(models_dir(), "SmolLM2-135M-Instruct-Q8_0.gguf") + if (!model_available(t, path)) { + return + } + // the converter's probed-config path: needs the tier armed (run.das --suite image-vulkan) + apply_box_profile_runtime() + let probed = dlim_config_current("q8") + if (!probed.vulkan_on) { + t |> skip("vulkan tier not armed (DASLLAMA_GPU) — run via run.das --suite image-vulkan") + return + } + if (!moe_gpu_dry_bake_arm(probed.vulkan)) { + t |> skip("dry bake declined to arm — see the log") + return + } + vulkan_bake_begin() + var m <- load_model_(path, QuantMode.q8) + t |> success(vulkan_bake_take(m), "the dry walk collected a bake") + t |> success(long_length(m.vkplan) > 0l, "plan has entries ({long_length(m.vkplan)})") + t |> success(long_length(m.vkblob) > 0l, "blob has bytes ({long_length(m.vkblob)})") + // every entry is role-stamped and points inside the blob + var roles_ok = true + var bounds_ok = true + for (e in m.vkplan) { + roles_ok = roles_ok && e.role != int(VkBakeRole.none) + bounds_ok = (bounds_ok && e.wq_off + e.wq_bytes <= long_length(m.vkblob) + && e.ws_off + e.ws_bytes <= long_length(m.vkblob)) + } + t |> success(roles_ok, "every plan entry carries a stage role") + t |> success(bounds_ok, "every plan entry's planes sit inside the blob") + // the flavor image round-trips the plan verbatim + var terr : string + let tmp = temp_directory(terr) + let img = path_join(empty(tmp) ? "." : tmp, "dasllama_vkbake_{ref_time_ticks()}.dlim") + let tag = moe_gpu_bake_tag() + t |> success(!empty(tag), "the armed tier names its bake tag") + t |> success(save_model_image(m, img, tag, "q8"), "flavor image saves") + var b = Model() + t |> success(load_model_image(img, b, tag), "flavor image maps") + t |> equal(long_length(m.vkplan), long_length(b.vkplan), "plan survives the round trip") + t |> equal(long_length(m.vkblob), long_length(b.vkblob), "blob survives the round trip") + delete b + delete m + remove(img) + } + } +} diff --git a/skills/babysit.md b/skills/babysit.md index 981211a1eb..035abf05ed 100644 --- a/skills/babysit.md +++ b/skills/babysit.md @@ -22,6 +22,14 @@ Never merge with an unresolved thread or with Copilot's latest review targeting ## 1. Watching the PR — the loop +**YOU babysit the build — the user does not babysit YOU.** A ready, verified fix gets +committed and pushed **immediately**. "Batch fixes into one push" means: fix every *distinct* +failure currently on the table, then push once — it does **NOT** mean waiting for the rest of +the matrix to finish. Straggler lanes reproducing an already-fixed failure are zero +information (the post-push run re-tests everything); idling on them while holding ready fixes +is the anti-pattern. If two consecutive status updates say "still waiting", you are doing it +wrong — act. + **Iterate against Copilot (~5 min), not the CI matrix (~30 min).** Copilot's review is a free ruleset check that lands in ~5 minutes; the full CI matrix is free too but takes ~30. So the loop is driven by Copilot and you **never sit through a full matrix between rounds**: 1. Compare the PR's current `headRefOid` with Copilot's latest review `commit_id`. diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 796e5beab0..00182e0b91 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -43,6 +43,21 @@ MAKE_TYPE_FACTORY(clock, das::Time)// use MAKE_TYPE_FACTORY out of namespace. So #define getchar_wrapper getchar #endif +// The direct sequential writer (dwrite_*): strictly-ascending append for bulk producers that +// already know their total size — measured 2060 MB/s vs 275 MB/s for the same volume through +// buffered fwrite. Cache-bypassing where the platform offers it (FILE_FLAG_NO_BUFFERING on +// Windows, F_NOCACHE on macOS); the generic POSIX arm is buffered write + posix_fadvise +// DONTNEED — the page cache is still touched, just released, so the producer's READS keep it. +// Implemented per-platform at the bottom of this file (same split as the mmap shim above); the +// handle owns an aligned bounce buffer, so callers append any pointer at any size. +void * das_dwrite_open ( const char * path, uint64_t total_bytes, uint64_t band_bytes ); +bool das_dwrite_append ( void * h, const void * data, uint64_t bytes ); +void * das_dwrite_band ( void * h, uint64_t * avail ); +bool das_dwrite_commit ( void * h, uint64_t bytes ); +uint64_t das_dwrite_stat ( void * h, int which ); +bool das_dwrite_close ( void * h ); +bool das_prefetch_map ( void * base, uint64_t bytes ); + namespace das { struct TimeAnnotation : ManagedValueAnnotation