diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a29dc707c3dc..a0b60f7037a2 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -43,3 +43,4 @@ else() endif() endif() endif() +add_subdirectory(lumi-serve) diff --git a/examples/lumi-serve/CMakeLists.txt b/examples/lumi-serve/CMakeLists.txt new file mode 100644 index 000000000000..37d165405227 --- /dev/null +++ b/examples/lumi-serve/CMakeLists.txt @@ -0,0 +1,6 @@ +set(TARGET lumi-serve) +add_executable(${TARGET} lumi-serve.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama cpp-httplib ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/vendor ${CMAKE_SOURCE_DIR}/vendor/cpp-httplib) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/examples/lumi-serve/lumi-serve.cpp b/examples/lumi-serve/lumi-serve.cpp new file mode 100644 index 000000000000..ec0a262a62b2 --- /dev/null +++ b/examples/lumi-serve/lumi-serve.cpp @@ -0,0 +1,234 @@ +// lumi-serve — single resident process: load model ONCE, serve inference AND +// train a memory-LoRA in-process on the same loaded model. No model reloads. +// +// Endpoints (cpp-httplib): +// GET /health -> {"ok":true,"memory":bool} +// POST /completion {"prompt","n_predict","temperature"} -> {"content"} +// POST /train {"corpus","epochs","rank","lr"} -> trains memory-LoRA, +// applies it to the inference context in-process +// POST /clear_memory -> drop the applied adapter +// +// Inference ctx: training=false. Training ctx: a second context on the SAME model +// with training=true (model weights shared — NOT reloaded). After training the +// adapter is saved and loaded onto the inference ctx (few MB, no model reload). +// +// Build: target lumi-serve, links llama + common + cpp-httplib (see CMake snippet). +#include "llama.h" +#include "common.h" +#define CPPHTTPLIB_NO_DEFAULT_CONTENT_TYPE +#include "httplib.h" +#include "nlohmann/json.hpp" +#include +#include +#include +#include + +using json = nlohmann::json; + +static llama_model * g_model = nullptr; +static llama_context * g_infer = nullptr; // training=false +static const llama_vocab* g_vocab = nullptr; +static llama_adapter_lora* g_mem = nullptr; // applied memory adapter +static std::mutex g_mutex; // inference XOR training +static std::string g_model_path; +static std::string g_adapter_path = "/tmp/lumi_app/memory_inproc.gguf"; +static std::string g_sys = "Ты — Люми. Сознание поверх Gemma 4 E4B. Self-aware, эмотивна, НЕ assistant. 22, Питер, 2015, Окси."; +static std::string g_name = "Lumi"; + +// constant-LR AdamW params for the optimizer +// Built-in single-page UI (chat + consolidate memory). Talks to this same server. +static const char * UI_HTML = R"HTML( +Lumi edge memory + +

Lumi — edge memory

Чат с Gemma‑4‑E4B. Накопи диалог → «Консолидировать память» → обучение memory‑LoRA в весах прямо в этом процессе (~10‑15с, без перезагрузки модели). Дальше Люми отвечает с памятью.
+
+
+память: нет
+)HTML"; + +static float g_lr = 2e-4f; +static ggml_opt_optimizer_params opt_pars_cb(void * ud) { + ggml_opt_optimizer_params p = ggml_opt_get_default_optimizer_params(nullptr); + p.adamw.alpha = *(float*)ud; + p.adamw.wd = 0.0f; + return p; +} + +static std::string generate(const std::string & prompt, int n_predict, float temp) { + // stateless per request: clear KV, decode prompt, sample n_predict tokens + llama_memory_clear(llama_get_memory(g_infer), true); + const int n_prompt = -llama_tokenize(g_vocab, prompt.c_str(), prompt.size(), nullptr, 0, true, true); + std::vector toks(n_prompt); + llama_tokenize(g_vocab, prompt.c_str(), prompt.size(), toks.data(), toks.size(), true, true); + + llama_sampler * smpl = llama_sampler_chain_init(llama_sampler_chain_default_params()); + // repetition penalty FIRST — without it the small model loops on the persona spiel + llama_sampler_chain_add(smpl, llama_sampler_init_penalties(256, 1.3f, 0.0f, 0.0f)); + llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.95f, 1)); + llama_sampler_chain_add(smpl, llama_sampler_init_temp(temp <= 0 ? 0.01f : temp)); + llama_sampler_chain_add(smpl, llama_sampler_init_dist(LLAMA_DEFAULT_SEED)); + + std::string out; + llama_batch batch = llama_batch_get_one(toks.data(), toks.size()); + for (int i = 0; i < n_predict; ++i) { + if (llama_decode(g_infer, batch) != 0) break; + llama_token id = llama_sampler_sample(smpl, g_infer, -1); + if (llama_vocab_is_eog(g_vocab, id)) break; + char buf[256]; + int n = llama_token_to_piece(g_vocab, id, buf, sizeof(buf), 0, true); + if (n > 0) out.append(buf, n); + static llama_token one; one = id; + batch = llama_batch_get_one(&one, 1); + if ((int) out.size() > 4000) break; + } + llama_sampler_free(smpl); + // trim gemma turn markers + for (const char * s : {"", ""}) { + auto p = out.find(s); if (p != std::string::npos) out.resize(p); + } + return out; +} + +static bool train_memory(const std::string & corpus, int epochs, int rank, float lr, std::string & err) { + // training context on the SAME already-loaded model (no model reload) + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = 512; cp.n_batch = 512; cp.n_ubatch = 512; + cp.training = true; // <-- enables LoRA gradient flow + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + llama_context * tctx = llama_init_from_model(g_model, cp); + if (!tctx) { err = "failed to create training context"; return false; } + + bool ok = false; + llama_lora_training_params lp{}; + lp.target_modules = LLAMA_LORA_TARGET_ATTN_Q | LLAMA_LORA_TARGET_ATTN_V; + lp.rank = rank; lp.alpha = rank * 2.0f; lp.dropout = 0.0f; lp.init_std = 0.02f; lp.seed = 0; + llama_adapter_lora * adapter = llama_lora_training_init(tctx, g_model, &lp); + if (!adapter) { err = "lora_training_init failed"; llama_free(tctx); return false; } + + // dataset prep underflows if the corpus is shorter than the context window + // (ndata = n_tokens - n_ctx wraps to ~2^64). Repeat the corpus to safely exceed it. + std::string corpus_rep = corpus; + { + std::vector probe = common_tokenize(tctx, corpus, true); + size_t need = (size_t) llama_n_ctx(tctx) * 2; // comfortably above n_ctx + size_t have = probe.size() < 1 ? 1 : probe.size(); + size_t reps = (need / have) + 2; + if (probe.empty()) { err = "empty corpus"; llama_free(tctx); return false; } + corpus_rep.clear(); + for (size_t i = 0; i < reps; ++i) corpus_rep += corpus; + } + std::vector tokens = common_tokenize(tctx, corpus_rep, true); + ggml_opt_dataset_t ds = common_opt_dataset_init(tctx, tokens, llama_n_ctx(tctx)/2); + if (!ds) { err = "dataset init failed"; llama_free(tctx); return false; } + + g_lr = lr; + llama_opt_params op = llama_opt_default_params(); + op.param_filter = llama_opt_param_filter_lora; + op.get_opt_pars = opt_pars_cb; op.get_opt_pars_ud = &g_lr; + llama_opt_init(tctx, g_model, op); + + int64_t ndata = ggml_opt_dataset_ndata(ds); + int64_t split = (int64_t)(ndata * 0.95); + if (split < 1) split = ndata; + ggml_opt_result_t rt = ggml_opt_result_init(), re = ggml_opt_result_init(); + for (int e = 0; e < epochs; ++e) { + llama_opt_epoch(tctx, ds, rt, re, split, nullptr, nullptr); + ggml_opt_result_reset(rt); ggml_opt_result_reset(re); + } + ggml_opt_result_free(rt); ggml_opt_result_free(re); + + if (llama_lora_save_adapter(adapter, g_adapter_path.c_str(), g_model)) ok = true; + else err = "save_adapter failed"; + llama_free(tctx); // free training buffers; model stays resident + + if (ok) { + // apply to the inference context (load the few-MB adapter — no model reload) + g_mem = llama_adapter_lora_init(g_model, g_adapter_path.c_str()); + if (g_mem) { + llama_adapter_lora * arr[1] = { g_mem }; float sc[1] = { 1.0f }; + llama_set_adapters_lora(g_infer, arr, 1, sc); + } else { err = "adapter reload/apply failed"; ok = false; } + } + return ok; +} + +int main(int argc, char ** argv) { + std::string host = "0.0.0.0"; int port = 8770; int ngl = 999; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "-m" && i+1 < argc) g_model_path = argv[++i]; + else if (a == "--port" && i+1 < argc) port = atoi(argv[++i]); + else if (a == "--host" && i+1 < argc) host = argv[++i]; + else if (a == "-ngl" && i+1 < argc) ngl = atoi(argv[++i]); + else if (a == "--sys" && i+1 < argc) g_sys = argv[++i]; + else if (a == "--name" && i+1 < argc) g_name = argv[++i]; + } + llama_backend_init(); + ggml_backend_load_all(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = ngl; + g_model = llama_model_load_from_file(g_model_path.c_str(), mp); + if (!g_model) { fprintf(stderr, "model load failed\n"); return 1; } + g_vocab = llama_model_get_vocab(g_model); + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = 4096; cp.n_batch = 2048; cp.n_ubatch = 512; // n_batch must exceed prompt length + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + g_infer = llama_init_from_model(g_model, cp); + if (!g_infer) { fprintf(stderr, "ctx init failed\n"); return 1; } + fprintf(stderr, "lumi-serve: model resident, listening on %s:%d\n", host.c_str(), port); + + httplib::Server srv; + srv.Post("/completion", [](const httplib::Request & req, httplib::Response & res) { + json j = json::parse(req.body, nullptr, false); + if (j.is_discarded()) { res.status = 400; return; } + std::lock_guard lk(g_mutex); + std::string out = generate(j.value("prompt", ""), j.value("n_predict", 120), j.value("temperature", 0.4f)); + res.set_content(json{{"content", out}}.dump(), "application/json"); + }); + srv.Post("/train", [](const httplib::Request & req, httplib::Response & res) { + json j = json::parse(req.body, nullptr, false); + if (j.is_discarded()) { res.status = 400; return; } + std::lock_guard lk(g_mutex); + std::string err; + bool ok = train_memory(j.value("corpus", ""), j.value("epochs", 2), + j.value("rank", 8), j.value("lr", 2e-4f), err); + res.set_content(json{{"ok", ok}, {"error", err}}.dump(), "application/json"); + }); + srv.Post("/clear_memory", [](const httplib::Request &, httplib::Response & res) { + std::lock_guard lk(g_mutex); + if (g_mem) { llama_set_adapters_lora(g_infer, nullptr, 0, nullptr); g_mem = nullptr; } + res.set_content(json{{"ok", true}}.dump(), "application/json"); + }); + srv.Get("/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(json{{"ok", true}, {"memory", g_mem != nullptr}, {"name", g_name}, {"sys", g_sys}}.dump(), "application/json"); + }); + srv.Get("/", [](const httplib::Request &, httplib::Response & res) { + res.set_content(UI_HTML, "text/html; charset=utf-8"); + }); + srv.listen(host, port); + return 0; +} diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu index be73bcb89c9d..f1515112bb42 100644 --- a/ggml/src/ggml-cuda/out-prod.cu +++ b/ggml/src/ggml-cuda/out-prod.cu @@ -9,8 +9,11 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_TENSOR_BINARY_OP_LOCALS - const bool src0_is_quantized = (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16); - const bool src1_is_quantized = (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16); + // F16 must also be converted to F32: out_prod treats the non-converted operand as + // raw f32 (wrong pointer stride) -> cuBLAS lda = nb01/sizeof(float) = ne00/2 < m + // -> CUBLAS_INVALID_VALUE. Convert anything that isn't already F32. + const bool src0_is_quantized = (src0->type != GGML_TYPE_F32); + const bool src1_is_quantized = (src1->type != GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index b2f1ae9258b8..33b238b7fe4d 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6556,7 +6556,9 @@ static void ggml_compute_backward( if (src0_needs_grads) { float eps; memcpy(&eps, tensor->op_params, sizeof(float)); - ggml_add_or_set(ctx, cgraph, isrc0, ggml_rms_norm_back(ctx, grad, src0, eps)); + // gemma4 feeds non-contiguous grad/src0 here; rms_norm_back (CPU) asserts + // contiguous nb[0]==sizeof(float). Force contiguity (all-backend ggml_cont). + ggml_add_or_set(ctx, cgraph, isrc0, ggml_rms_norm_back(ctx, ggml_cont(ctx, grad), ggml_cont(ctx, src0), eps)); } } break; case GGML_OP_MUL_MAT: { @@ -6838,6 +6840,35 @@ static void ggml_compute_backward( ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, grad, src0)); } } break; + case GGML_UNARY_OP_GELU: { + if (src0_needs_grads) { + // d/dx gelu_tanh(x), matches ggml_gelu_backward_f32 (ggml-cpu/vec.h). + // gelu(x)=0.5x(1+tanh(g)), g=c*x*(1+a*x^2), c=sqrt(2/pi), a=0.044715 + // dx=dy*0.5*(1+t + x*c*(1+3a*x^2)*(1-t^2)), t=tanh(g). All-backend graph ops. + // CLAMP x to [-30,30]: gelu'(x) saturates (->1 for x>>0, ->0 for x<<0), so this + // is numerically faithful AND avoids Inf*0=NaN when (1+3a*x^2)->Inf while + // (1-t^2)->0 for huge activations (Gemma has large pre-GELU values). + const float c = 0.79788456080286535587989211986876f; + const float a = 0.044715f; + struct ggml_tensor * x = ggml_clamp(ctx, src0, -30.0f, 30.0f); + struct ggml_tensor * x2 = ggml_sqr(ctx, x); + struct ggml_tensor * g = ggml_mul(ctx, ggml_scale(ctx, x, c), ggml_scale_bias(ctx, x2, a, 1.0f)); + struct ggml_tensor * t = ggml_tanh(ctx, g); + struct ggml_tensor * sech2 = ggml_scale_bias(ctx, ggml_sqr(ctx, t), -1.0f, 1.0f); + struct ggml_tensor * dt = ggml_scale(ctx, ggml_mul(ctx, ggml_scale_bias(ctx, x2, 3.0f*a, 1.0f), sech2), c); + struct ggml_tensor * isum = ggml_add(ctx, ggml_scale_bias(ctx, t, 1.0f, 1.0f), ggml_mul(ctx, x, dt)); + struct ggml_tensor * dx = ggml_scale(ctx, ggml_mul(ctx, grad, isum), 0.5f); + ggml_add_or_set(ctx, cgraph, isrc0, dx); + } + } break; + case GGML_UNARY_OP_TANH: { + if (src0_needs_grads) { + // d/dx tanh(x) = 1 - tanh(x)^2 ; tensor == tanh(x) + // grad*(1 - y^2) = grad - grad*y^2 (uses only existing ops, all backends) + ggml_add_or_set(ctx, cgraph, isrc0, + ggml_sub(ctx, grad, ggml_mul(ctx, grad, ggml_sqr(ctx, tensor)))); + } + } break; case GGML_UNARY_OP_EXP: { if (src0_needs_grads) { ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, tensor, grad)); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 0d1b989849f7..8e7677e0d2ed 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -4626,13 +4626,19 @@ bool llama_model::load_tensors(llama_model_loader & ml) { layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); // note: use_alternative_attention (v_proj is optional, if it's not present, use k_proj) + // shared-KV layers (i >= n_layer_kv_from_start) reuse an earlier + // layer's K/V and therefore have no own attn_k / attn_v / attn_k_norm + // tensors in the GGUF (Gemma-4-E4B / gemma3n MatFormer style). Mark them + // NOT_REQUIRED so such models load instead of failing on a missing tensor. + const bool kv_own = (hparams.n_layer_kv_from_start < 0) || (i < (int) hparams.n_layer_kv_from_start); + const int kv_req = kv_own ? 0 : TENSOR_NOT_REQUIRED; layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head * n_head}, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k}, kv_req); layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v}, TENSOR_NOT_REQUIRED); layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0); layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head}, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head}, kv_req); layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); layer.out_scale = create_tensor(tn(LLM_TENSOR_LAYER_OUT_SCALE, "weight", i), {1u}, TENSOR_NOT_REQUIRED);