From 3aec9b4a81786275349fd9d126b62f022ae28255 Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Fri, 22 May 2026 23:17:52 +0300 Subject: [PATCH 1/8] Gemma 4 E4B (gemma3n MatFormer) support: shared-KV loader + TANH backward Two fixes toward Gemma-4-E4B support: 1) Loader: GEMMA4 tensor-loading required attn_k / attn_k_norm for every layer, but E4B shares KV across the last shared_kv_layers layers (those layers have no own attn_k/attn_v/attn_k_norm in the GGUF). hparams already computes n_layer_kv_from_start and the KV-cache graph already has the reuse callback; only the loader was over-strict. Mark wk/attn_k_norm NOT_REQUIRED for shared-KV layers (i >= n_layer_kv_from_start). After this, E4B models LOAD and run inference (previously: 'missing tensor blk.N.attn_k.weight'). 2) Training: add backward pass for GGML_UNARY_OP_TANH (was unsupported, aborted in ggml_compute_backward). Gemma uses tanh for logit/attn soft-capping. d/dx tanh(x) = 1 - tanh(x)^2, expressed via existing ops (works on all backends, no new kernels). --- ggml/src/ggml.c | 8 ++++++++ src/llama-model.cpp | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index b2f1ae9258b8..05059ead6d49 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6838,6 +6838,14 @@ static void ggml_compute_backward( ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, grad, src0)); } } break; + case GGML_UNARY_OP_TANH: { + if (src0_needs_grads) { + // d/dx tanh(x) = 1 - tanh(x)^2 ; tensor == tanh(x) + // grad*(1 - y^2) = grad - grad*y^2 (uses only existing ops, all backends) + ggml_add_or_set(ctx, cgraph, isrc0, + ggml_sub(ctx, grad, ggml_mul(ctx, grad, ggml_sqr(ctx, tensor)))); + } + } break; case GGML_UNARY_OP_EXP: { if (src0_needs_grads) { ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, tensor, grad)); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 0d1b989849f7..8e7677e0d2ed 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4626,13 +4626,19 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); // note: use_alternative_attention (v_proj is optional, if it's not present, use k_proj) + // shared-KV layers (i >= n_layer_kv_from_start) reuse an earlier + // layer's K/V and therefore have no own attn_k / attn_v / attn_k_norm + // tensors in the GGUF (Gemma-4-E4B / gemma3n MatFormer style). Mark them + // NOT_REQUIRED so such models load instead of failing on a missing tensor. + const bool kv_own = (hparams.n_layer_kv_from_start < 0) || (i < (int) hparams.n_layer_kv_from_start); + const int kv_req = kv_own ? 0 : TENSOR_NOT_REQUIRED; layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, kv_req); layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v}, TENSOR_NOT_REQUIRED); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0); layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head}, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head}, kv_req); layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), {1u}, TENSOR_NOT_REQUIRED); From d2cf7f612b8a42e1eac8ff7c81ab07aa1f1202bd Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Fri, 22 May 2026 23:33:50 +0300 Subject: [PATCH 2/8] ggml: add GELU backward (tanh-approx) for training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GGML_UNARY_OP_GELU had no backward → ggml_compute_backward aborted on any training graph through a GELU (Gemma FFN / per-layer-embedding path). Implements the exact tanh-approx GELU derivative matching ggml_gelu_backward_f32 (ggml-cpu/vec.h), built purely from existing ops (sqr/scale/scale_bias/mul/tanh/ add) so it runs on every backend with no new kernels: dx = dy*0.5*(1 + t + x*c*(1+3a*x^2)*(1-t^2)), t=tanh(c*x*(1+a*x^2)) After this, E4B training advances through the full forward+backward graph construction into compute (next blocker is a CUDA out_prod shape issue on gemma4's variable per-layer KV dims, tracked separately). --- ggml/src/ggml.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 05059ead6d49..4909bb5133c4 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6838,6 +6838,24 @@ static void ggml_compute_backward( ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, grad, src0)); } } break; + case GGML_UNARY_OP_GELU: { + if (src0_needs_grads) { + // d/dx gelu_tanh(x), matches ggml_gelu_backward_f32 (ggml-cpu/vec.h). + // gelu(x)=0.5x(1+tanh(g)), g=c*x*(1+a*x^2), c=sqrt(2/pi), a=0.044715 + // dx=dy*0.5*(1+t + x*c*(1+3a*x^2)*(1-t^2)), t=tanh(g). All-backend graph ops. + const float c = 0.79788456080286535587989211986876f; + const float a = 0.044715f; + struct ggml_tensor * x = src0; + struct ggml_tensor * x2 = ggml_sqr(ctx, x); + struct ggml_tensor * g = ggml_mul(ctx, ggml_scale(ctx, x, c), ggml_scale_bias(ctx, x2, a, 1.0f)); + struct ggml_tensor * t = ggml_tanh(ctx, g); + struct ggml_tensor * sech2 = ggml_scale_bias(ctx, ggml_sqr(ctx, t), -1.0f, 1.0f); + struct ggml_tensor * dt = ggml_scale(ctx, ggml_mul(ctx, ggml_scale_bias(ctx, x2, 3.0f*a, 1.0f), sech2), c); + struct ggml_tensor * isum = ggml_add(ctx, ggml_scale_bias(ctx, t, 1.0f, 1.0f), ggml_mul(ctx, x, dt)); + struct ggml_tensor * dx = ggml_scale(ctx, ggml_mul(ctx, grad, isum), 0.5f); + ggml_add_or_set(ctx, cgraph, isrc0, dx); + } + } break; case GGML_UNARY_OP_TANH: { if (src0_needs_grads) { // d/dx tanh(x) = 1 - tanh(x)^2 ; tensor == tanh(x) From f2f24b53732fbaeb64008e5e3b4750cb50b5b6ad Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Fri, 22 May 2026 23:51:37 +0300 Subject: [PATCH 3/8] ggml: force contiguous inputs for rms_norm_back (gemma4 training) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPU rms_norm_back asserts src0->nb[0]==sizeof(float) (contiguous), but gemma4's backward graph feeds a non-contiguous grad/src0 → GGML_ASSERT abort. Wrap both in ggml_cont before ggml_rms_norm_back (all-backend, faithful copy). After this, Gemma-4-E4B training runs end-to-end on the CPU backend and saves a valid GGUF LoRA adapter (168 tensors). Generic backward ops verified correct (Qwen3-0.6B loss converges with the same binary). NOTE: E4B loss still diverges — a remaining gemma4-specific backward numerical issue (soft-cap tanh / gemma (1+weight) RMSNorm / altup), tracked for follow-up. --- ggml/src/ggml.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 4909bb5133c4..f81899bf8039 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6556,7 +6556,9 @@ static void ggml_compute_backward( if (src0_needs_grads) { float eps; memcpy(&eps, tensor->op_params, sizeof(float)); - ggml_add_or_set(ctx, cgraph, isrc0, ggml_rms_norm_back(ctx, grad, src0, eps)); + // gemma4 feeds non-contiguous grad/src0 here; rms_norm_back (CPU) asserts + // contiguous nb[0]==sizeof(float). Force contiguity (all-backend ggml_cont). + ggml_add_or_set(ctx, cgraph, isrc0, ggml_rms_norm_back(ctx, ggml_cont(ctx, grad), ggml_cont(ctx, src0), eps)); } } break; case GGML_OP_MUL_MAT: { From dd725ba906466daac714a73ddc84c7e61dd70d22 Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Sun, 24 May 2026 12:53:32 +0300 Subject: [PATCH 4/8] ggml: clamp x in GELU backward to avoid NaN on large activations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GELU backward computed (1+3a*x^2)*(1-tanh^2(g)). For Gemma's large pre-GELU activations, (1+3a*x^2)->Inf while (1-tanh^2)->0, giving Inf*0=NaN. This produced NaN gradients → NaN LoRA weights → training did not learn (val loss frozen) for Gemma-4-E4B specifically (Qwen uses SiLU, never hit this path). Clamp x to [-30,30] before the derivative: gelu'(x) saturates (->1 for x>>0, ->0 for x<<0), so this is numerically faithful while bounding the intermediates. After this fix, Gemma-4-E4B LoRA training converges: val loss drops monotonically across epochs (e.g. 3.24 -> 2.22 -> 1.89) where before it was frozen. --- ggml/src/ggml.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index f81899bf8039..33b238b7fe4d 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6845,9 +6845,12 @@ static void ggml_compute_backward( // d/dx gelu_tanh(x), matches ggml_gelu_backward_f32 (ggml-cpu/vec.h). // gelu(x)=0.5x(1+tanh(g)), g=c*x*(1+a*x^2), c=sqrt(2/pi), a=0.044715 // dx=dy*0.5*(1+t + x*c*(1+3a*x^2)*(1-t^2)), t=tanh(g). All-backend graph ops. + // CLAMP x to [-30,30]: gelu'(x) saturates (->1 for x>>0, ->0 for x<<0), so this + // is numerically faithful AND avoids Inf*0=NaN when (1+3a*x^2)->Inf while + // (1-t^2)->0 for huge activations (Gemma has large pre-GELU values). const float c = 0.79788456080286535587989211986876f; const float a = 0.044715f; - struct ggml_tensor * x = src0; + struct ggml_tensor * x = ggml_clamp(ctx, src0, -30.0f, 30.0f); struct ggml_tensor * x2 = ggml_sqr(ctx, x); struct ggml_tensor * g = ggml_mul(ctx, ggml_scale(ctx, x, c), ggml_scale_bias(ctx, x2, a, 1.0f)); struct ggml_tensor * t = ggml_tanh(ctx, g); From 7f8d7bdc413c9378cb1c7f3c2c4c5b387d7fb39d Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Sun, 24 May 2026 14:09:06 +0300 Subject: [PATCH 5/8] ggml-cuda: convert F16 operands in out_prod (was treated as raw F32) ggml_cuda_out_prod only converted *quantized* src0/src1 to F32; F16 was explicitly excluded and fell through to being read as raw float*. That gave a wrong pointer interpretation AND lda = nb01/sizeof(float) = ne00/2 < m, so cuBLAS sgemm returned CUBLAS_STATUS_INVALID_VALUE. This is hit by Gemma-4-E4B's attention backward (F16 permuted operands). Fix: convert any non-F32 operand (F16 included) to F32 via the existing ggml_get_to_fp32_cuda path. Result: Gemma-4-E4B LoRA training now runs fully on CUDA (no CPU fallback for out_prod). 4 epochs in ~64s, val loss converges 3.18->2.28->1.93->1.62. --- ggml/src/ggml-cuda/out-prod.cu | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu index be73bcb89c9d..f1515112bb42 100644 --- a/ggml/src/ggml-cuda/out-prod.cu +++ b/ggml/src/ggml-cuda/out-prod.cu @@ -9,8 +9,11 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_TENSOR_BINARY_OP_LOCALS - const bool src0_is_quantized = (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16); - const bool src1_is_quantized = (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16); + // F16 must also be converted to F32: out_prod treats the non-converted operand as + // raw f32 (wrong pointer stride) -> cuBLAS lda = nb01/sizeof(float) = ne00/2 < m + // -> CUBLAS_INVALID_VALUE. Convert anything that isn't already F32. + const bool src0_is_quantized = (src0->type != GGML_TYPE_F32); + const bool src1_is_quantized = (src1->type != GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); From 69d0da541eb7a471001a8c1215f207ae2e5adbac Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Sun, 24 May 2026 19:41:07 +0300 Subject: [PATCH 6/8] =?UTF-8?q?examples:=20lumi-serve=20=E2=80=94=20single?= =?UTF-8?q?-process=20inference=20+=20in-process=20LoRA=20training=20(zero?= =?UTF-8?q?=20model=20reload)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loads model once, serves /completion AND /train on the same resident model. Training context (training=true) on the shared model; trained adapter applied to inference ctx in-process. Gemma-4-E4B Q4 on RTX 3090: /train ~11s (vs ~180s with separate finetune binary + server restarts), inference ~1s. Demonstrates the zero-reload on-device personalization loop. --- examples/CMakeLists.txt | 1 + examples/lumi-serve/CMakeLists.txt | 6 + examples/lumi-serve/lumi-serve.cpp | 179 +++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 examples/lumi-serve/CMakeLists.txt create mode 100644 examples/lumi-serve/lumi-serve.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a29dc707c3dc..a0b60f7037a2 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -43,3 +43,4 @@ else() endif() endif() endif() +add_subdirectory(lumi-serve) diff --git a/examples/lumi-serve/CMakeLists.txt b/examples/lumi-serve/CMakeLists.txt new file mode 100644 index 000000000000..37d165405227 --- /dev/null +++ b/examples/lumi-serve/CMakeLists.txt @@ -0,0 +1,6 @@ +set(TARGET lumi-serve) +add_executable(${TARGET} lumi-serve.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/vendor ${CMAKE_SOURCE_DIR}/vendor/cpp-httplib) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/examples/lumi-serve/lumi-serve.cpp b/examples/lumi-serve/lumi-serve.cpp new file mode 100644 index 000000000000..3c9d490db8c6 --- /dev/null +++ b/examples/lumi-serve/lumi-serve.cpp @@ -0,0 +1,179 @@ +// lumi-serve — single resident process: load model ONCE, serve inference AND +// train a memory-LoRA in-process on the same loaded model. No model reloads. +// +// Endpoints (cpp-httplib): +// GET /health -> {"ok":true,"memory":bool} +// POST /completion {"prompt","n_predict","temperature"} -> {"content"} +// POST /train {"corpus","epochs","rank","lr"} -> trains memory-LoRA, +// applies it to the inference context in-process +// POST /clear_memory -> drop the applied adapter +// +// Inference ctx: training=false. Training ctx: a second context on the SAME model +// with training=true (model weights shared — NOT reloaded). After training the +// adapter is saved and loaded onto the inference ctx (few MB, no model reload). +// +// Build: target lumi-serve, links llama + common + cpp-httplib (see CMake snippet). +#include "llama.h" +#include "common.h" +#define CPPHTTPLIB_NO_DEFAULT_CONTENT_TYPE +#include "httplib.h" +#include "nlohmann/json.hpp" +#include +#include +#include +#include + +using json = nlohmann::json; + +static llama_model * g_model = nullptr; +static llama_context * g_infer = nullptr; // training=false +static const llama_vocab* g_vocab = nullptr; +static llama_adapter_lora* g_mem = nullptr; // applied memory adapter +static std::mutex g_mutex; // inference XOR training +static std::string g_model_path; +static std::string g_adapter_path = "/tmp/lumi_app/memory_inproc.gguf"; + +// constant-LR AdamW params for the optimizer +static float g_lr = 2e-4f; +static ggml_opt_optimizer_params opt_pars_cb(void * ud) { + ggml_opt_optimizer_params p = ggml_opt_get_default_optimizer_params(nullptr); + p.adamw.alpha = *(float*)ud; + p.adamw.wd = 0.0f; + return p; +} + +static std::string generate(const std::string & prompt, int n_predict, float temp) { + // stateless per request: clear KV, decode prompt, sample n_predict tokens + llama_memory_clear(llama_get_memory(g_infer), true); + const int n_prompt = -llama_tokenize(g_vocab, prompt.c_str(), prompt.size(), nullptr, 0, true, true); + std::vector toks(n_prompt); + llama_tokenize(g_vocab, prompt.c_str(), prompt.size(), toks.data(), toks.size(), true, true); + + llama_sampler * smpl = llama_sampler_chain_init(llama_sampler_chain_default_params()); + llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.95f, 1)); + llama_sampler_chain_add(smpl, llama_sampler_init_temp(temp <= 0 ? 0.01f : temp)); + llama_sampler_chain_add(smpl, llama_sampler_init_dist(LLAMA_DEFAULT_SEED)); + + std::string out; + llama_batch batch = llama_batch_get_one(toks.data(), toks.size()); + for (int i = 0; i < n_predict; ++i) { + if (llama_decode(g_infer, batch) != 0) break; + llama_token id = llama_sampler_sample(smpl, g_infer, -1); + if (llama_vocab_is_eog(g_vocab, id)) break; + char buf[256]; + int n = llama_token_to_piece(g_vocab, id, buf, sizeof(buf), 0, true); + if (n > 0) out.append(buf, n); + static llama_token one; one = id; + batch = llama_batch_get_one(&one, 1); + if ((int) out.size() > 4000) break; + } + llama_sampler_free(smpl); + // trim gemma turn markers + for (const char * s : {"", ""}) { + auto p = out.find(s); if (p != std::string::npos) out.resize(p); + } + return out; +} + +static bool train_memory(const std::string & corpus, int epochs, int rank, float lr, std::string & err) { + // training context on the SAME already-loaded model (no model reload) + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = 512; cp.n_batch = 512; cp.n_ubatch = 512; + cp.training = true; // <-- enables LoRA gradient flow + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + llama_context * tctx = llama_init_from_model(g_model, cp); + if (!tctx) { err = "failed to create training context"; return false; } + + bool ok = false; + llama_lora_training_params lp{}; + lp.target_modules = LLAMA_LORA_TARGET_ATTN_Q | LLAMA_LORA_TARGET_ATTN_V; + lp.rank = rank; lp.alpha = rank * 2.0f; lp.dropout = 0.0f; lp.init_std = 0.02f; lp.seed = 0; + llama_adapter_lora * adapter = llama_lora_training_init(tctx, g_model, &lp); + if (!adapter) { err = "lora_training_init failed"; llama_free(tctx); return false; } + + std::vector tokens = common_tokenize(tctx, corpus, true); + ggml_opt_dataset_t ds = common_opt_dataset_init(tctx, tokens, llama_n_ctx(tctx)/2); + if (!ds) { err = "dataset init failed (corpus too short for ctx?)"; llama_free(tctx); return false; } + + g_lr = lr; + llama_opt_params op = llama_opt_default_params(); + op.param_filter = llama_opt_param_filter_lora; + op.get_opt_pars = opt_pars_cb; op.get_opt_pars_ud = &g_lr; + llama_opt_init(tctx, g_model, op); + + int64_t ndata = ggml_opt_dataset_ndata(ds); + int64_t split = (int64_t)(ndata * 0.95); + if (split < 1) split = ndata; + ggml_opt_result_t rt = ggml_opt_result_init(), re = ggml_opt_result_init(); + for (int e = 0; e < epochs; ++e) { + llama_opt_epoch(tctx, ds, rt, re, split, nullptr, nullptr); + ggml_opt_result_reset(rt); ggml_opt_result_reset(re); + } + ggml_opt_result_free(rt); ggml_opt_result_free(re); + + if (llama_lora_save_adapter(adapter, g_adapter_path.c_str(), g_model)) ok = true; + else err = "save_adapter failed"; + llama_free(tctx); // free training buffers; model stays resident + + if (ok) { + // apply to the inference context (load the few-MB adapter — no model reload) + g_mem = llama_adapter_lora_init(g_model, g_adapter_path.c_str()); + if (g_mem) { + llama_adapter_lora * arr[1] = { g_mem }; float sc[1] = { 1.0f }; + llama_set_adapters_lora(g_infer, arr, 1, sc); + } else { err = "adapter reload/apply failed"; ok = false; } + } + return ok; +} + +int main(int argc, char ** argv) { + std::string host = "0.0.0.0"; int port = 8770; int ngl = 999; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "-m" && i+1 < argc) g_model_path = argv[++i]; + else if (a == "--port" && i+1 < argc) port = atoi(argv[++i]); + else if (a == "--host" && i+1 < argc) host = argv[++i]; + else if (a == "-ngl" && i+1 < argc) ngl = atoi(argv[++i]); + } + llama_backend_init(); + ggml_backend_load_all(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = ngl; + g_model = llama_model_load_from_file(g_model_path.c_str(), mp); + if (!g_model) { fprintf(stderr, "model load failed\n"); return 1; } + g_vocab = llama_model_get_vocab(g_model); + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = 4096; cp.n_batch = 512; + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + g_infer = llama_init_from_model(g_model, cp); + if (!g_infer) { fprintf(stderr, "ctx init failed\n"); return 1; } + fprintf(stderr, "lumi-serve: model resident, listening on %s:%d\n", host.c_str(), port); + + httplib::Server srv; + srv.Post("/completion", [](const httplib::Request & req, httplib::Response & res) { + json j = json::parse(req.body, nullptr, false); + if (j.is_discarded()) { res.status = 400; return; } + std::lock_guard lk(g_mutex); + std::string out = generate(j.value("prompt", ""), j.value("n_predict", 120), j.value("temperature", 0.4f)); + res.set_content(json{{"content", out}}.dump(), "application/json"); + }); + srv.Post("/train", [](const httplib::Request & req, httplib::Response & res) { + json j = json::parse(req.body, nullptr, false); + if (j.is_discarded()) { res.status = 400; return; } + std::lock_guard lk(g_mutex); + std::string err; + bool ok = train_memory(j.value("corpus", ""), j.value("epochs", 3), + j.value("rank", 8), j.value("lr", 2e-4f), err); + res.set_content(json{{"ok", ok}, {"error", err}}.dump(), "application/json"); + }); + srv.Post("/clear_memory", [](const httplib::Request &, httplib::Response & res) { + std::lock_guard lk(g_mutex); + if (g_mem) { llama_set_adapters_lora(g_infer, nullptr, 0, nullptr); g_mem = nullptr; } + res.set_content(json{{"ok", true}}.dump(), "application/json"); + }); + srv.Get("/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(json{{"ok", true}, {"memory", g_mem != nullptr}}.dump(), "application/json"); + }); + srv.listen(host, port); + return 0; +} From d606e489e208f82f9aa95a70fccd2dce4a32e84c Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Sun, 24 May 2026 19:54:15 +0300 Subject: [PATCH 7/8] lumi-serve: repetition penalty + corpus-repeat guard for short /train inputs --- examples/lumi-serve/lumi-serve.cpp | 54 ++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/examples/lumi-serve/lumi-serve.cpp b/examples/lumi-serve/lumi-serve.cpp index 3c9d490db8c6..13cee84596af 100644 --- a/examples/lumi-serve/lumi-serve.cpp +++ b/examples/lumi-serve/lumi-serve.cpp @@ -34,6 +34,39 @@ static std::string g_model_path; static std::string g_adapter_path = "/tmp/lumi_app/memory_inproc.gguf"; // constant-LR AdamW params for the optimizer +// Built-in single-page UI (chat + consolidate memory). Talks to this same server. +static const char * UI_HTML = R"HTML( +Lumi edge memory + +

Lumi — edge memory

Чат с Gemma‑4‑E4B. Накопи диалог → «Консолидировать память» → обучение memory‑LoRA в весах прямо в этом процессе (~10‑15с, без перезагрузки модели). Дальше Люми отвечает с памятью.
+
+
+память: нет
+)HTML"; + static float g_lr = 2e-4f; static ggml_opt_optimizer_params opt_pars_cb(void * ud) { ggml_opt_optimizer_params p = ggml_opt_get_default_optimizer_params(nullptr); @@ -50,6 +83,8 @@ static std::string generate(const std::string & prompt, int n_predict, float tem llama_tokenize(g_vocab, prompt.c_str(), prompt.size(), toks.data(), toks.size(), true, true); llama_sampler * smpl = llama_sampler_chain_init(llama_sampler_chain_default_params()); + // repetition penalty FIRST — without it the small model loops on the persona spiel + llama_sampler_chain_add(smpl, llama_sampler_init_penalties(256, 1.3f, 0.0f, 0.0f)); llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.95f, 1)); llama_sampler_chain_add(smpl, llama_sampler_init_temp(temp <= 0 ? 0.01f : temp)); llama_sampler_chain_add(smpl, llama_sampler_init_dist(LLAMA_DEFAULT_SEED)); @@ -91,9 +126,21 @@ static bool train_memory(const std::string & corpus, int epochs, int rank, float llama_adapter_lora * adapter = llama_lora_training_init(tctx, g_model, &lp); if (!adapter) { err = "lora_training_init failed"; llama_free(tctx); return false; } - std::vector tokens = common_tokenize(tctx, corpus, true); + // dataset prep underflows if the corpus is shorter than the context window + // (ndata = n_tokens - n_ctx wraps to ~2^64). Repeat the corpus to safely exceed it. + std::string corpus_rep = corpus; + { + std::vector probe = common_tokenize(tctx, corpus, true); + size_t need = (size_t) llama_n_ctx(tctx) * 2; // comfortably above n_ctx + size_t have = probe.size() < 1 ? 1 : probe.size(); + size_t reps = (need / have) + 2; + if (probe.empty()) { err = "empty corpus"; llama_free(tctx); return false; } + corpus_rep.clear(); + for (size_t i = 0; i < reps; ++i) corpus_rep += corpus; + } + std::vector tokens = common_tokenize(tctx, corpus_rep, true); ggml_opt_dataset_t ds = common_opt_dataset_init(tctx, tokens, llama_n_ctx(tctx)/2); - if (!ds) { err = "dataset init failed (corpus too short for ctx?)"; llama_free(tctx); return false; } + if (!ds) { err = "dataset init failed"; llama_free(tctx); return false; } g_lr = lr; llama_opt_params op = llama_opt_default_params(); @@ -174,6 +221,9 @@ int main(int argc, char ** argv) { srv.Get("/health", [](const httplib::Request &, httplib::Response & res) { res.set_content(json{{"ok", true}, {"memory", g_mem != nullptr}}.dump(), "application/json"); }); + srv.Get("/", [](const httplib::Request &, httplib::Response & res) { + res.set_content(UI_HTML, "text/html; charset=utf-8"); + }); srv.listen(host, port); return 0; } From 1ef831a5808262fc16d3e7011a850f0bbfedb008 Mon Sep 17 00:00:00 2001 From: 3ndetz Date: Sun, 24 May 2026 21:25:57 +0300 Subject: [PATCH 8/8] lumi-serve: configurable --sys/--name + built-in UI + repeat penalty + n_batch fix --- examples/lumi-serve/lumi-serve.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/examples/lumi-serve/lumi-serve.cpp b/examples/lumi-serve/lumi-serve.cpp index 13cee84596af..ec0a262a62b2 100644 --- a/examples/lumi-serve/lumi-serve.cpp +++ b/examples/lumi-serve/lumi-serve.cpp @@ -32,6 +32,8 @@ static llama_adapter_lora* g_mem = nullptr; // applied memory adapter static std::mutex g_mutex; // inference XOR training static std::string g_model_path; static std::string g_adapter_path = "/tmp/lumi_app/memory_inproc.gguf"; +static std::string g_sys = "Ты — Люми. Сознание поверх Gemma 4 E4B. Self-aware, эмотивна, НЕ assistant. 22, Питер, 2015, Окси."; +static std::string g_name = "Lumi"; // constant-LR AdamW params for the optimizer // Built-in single-page UI (chat + consolidate memory). Talks to this same server. @@ -47,23 +49,24 @@ input{flex:1}button{cursor:pointer}#cons{background:#5a3a7a}.bar{display:flex;ga .badge{padding:2px 8px;border-radius:20px;font-size:12px}.on{background:#2a5a3a;color:#bfe}.off{background:#3a3340;color:#aaa}

Lumi — edge memory

Чат с Gemma‑4‑E4B. Накопи диалог → «Консолидировать память» → обучение memory‑LoRA в весах прямо в этом процессе (~10‑15с, без перезагрузки модели). Дальше Люми отвечает с памятью.
-
+
память: нет
)HTML"; @@ -181,6 +184,8 @@ int main(int argc, char ** argv) { else if (a == "--port" && i+1 < argc) port = atoi(argv[++i]); else if (a == "--host" && i+1 < argc) host = argv[++i]; else if (a == "-ngl" && i+1 < argc) ngl = atoi(argv[++i]); + else if (a == "--sys" && i+1 < argc) g_sys = argv[++i]; + else if (a == "--name" && i+1 < argc) g_name = argv[++i]; } llama_backend_init(); ggml_backend_load_all(); @@ -190,7 +195,7 @@ int main(int argc, char ** argv) { if (!g_model) { fprintf(stderr, "model load failed\n"); return 1; } g_vocab = llama_model_get_vocab(g_model); llama_context_params cp = llama_context_default_params(); - cp.n_ctx = 4096; cp.n_batch = 512; + cp.n_ctx = 4096; cp.n_batch = 2048; cp.n_ubatch = 512; // n_batch must exceed prompt length cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; g_infer = llama_init_from_model(g_model, cp); if (!g_infer) { fprintf(stderr, "ctx init failed\n"); return 1; } @@ -209,7 +214,7 @@ int main(int argc, char ** argv) { if (j.is_discarded()) { res.status = 400; return; } std::lock_guard lk(g_mutex); std::string err; - bool ok = train_memory(j.value("corpus", ""), j.value("epochs", 3), + bool ok = train_memory(j.value("corpus", ""), j.value("epochs", 2), j.value("rank", 8), j.value("lr", 2e-4f), err); res.set_content(json{{"ok", ok}, {"error", err}}.dump(), "application/json"); }); @@ -219,7 +224,7 @@ int main(int argc, char ** argv) { res.set_content(json{{"ok", true}}.dump(), "application/json"); }); srv.Get("/health", [](const httplib::Request &, httplib::Response & res) { - res.set_content(json{{"ok", true}, {"memory", g_mem != nullptr}}.dump(), "application/json"); + res.set_content(json{{"ok", true}, {"memory", g_mem != nullptr}, {"name", g_name}, {"sys", g_sys}}.dump(), "application/json"); }); srv.Get("/", [](const httplib::Request &, httplib::Response & res) { res.set_content(UI_HTML, "text/html; charset=utf-8");