agentgrep(feat[insights]): Wire the LiteRT-LM runtime for local Gemma summaries

why: The MVP could fetch a Gemma .litertlm artifact but the llm level only
ran Ollama, so a downloaded model could not actually produce a summary.
LiteRT-LM has an installable in-process runtime, so the llm level can run
a local Gemma model end-to-end without a daemon.

what:
- Add a litert-lm backend to the llm level, selected by --backend, that
  loads the cached .litertlm via litert_lm.Engine, streams the reply
  through the progress sink, and provisions the model on demand.
- Default the LiteRT token budget to 2048: the budget is the total
  prompt+output KV-cache size, and undersizing it surfaces as an opaque
  tensor-allocation failure rather than a clear message.
- Quiet the LiteRT-LM C++ runtime to ERROR so streamed output is not
  buried under model-metadata logging on stderr.
- Order llm backends by the requested --backend, declare the
  insights-llm-litert extra, keep litert_lm out of the import path, and
  cover the runtime and the not-provisioned path with injected-fake tests.

Author: tony

docs/cli/insights.md

@@ -50,7 +50,7 @@ and, where relevant, the installed embedding model.
 | `ml` | `agentgrep[insights-ml]` | TF-IDF + KMeans topic clusters |
 | `embeddings` | `agentgrep[insights-embeddings]` | semantic clusters, near-duplicate detection |
 | `index` | `agentgrep[insights-index]` | persistent tantivy + sqlite-vec hybrid index |
-| `llm` | `agentgrep[insights-llm]` | local Ollama narrative summary |
+| `llm` | `agentgrep[insights-llm]` or `agentgrep[insights-llm-litert]` | local narrative summary via Ollama or an in-process LiteRT-LM model |
 
 List the rungs and what is installed:
 
@@ -77,13 +77,26 @@ defaults to `tantivy` + `sqlite-vec`; pass `--index-backend lancedb` after
 installing `agentgrep[insights-index-lancedb]` for the single-store
 alternative.
 
-Stream a grounded local-LLM summary (shown as `text` because it requires a
-running Ollama daemon):
+Stream a grounded local-LLM summary. Two runtimes are wired: Ollama over
+local HTTP, and an in-process LiteRT-LM model (e.g. a Gemma `.litertlm`
+artifact). Both examples are shown as `text` because they need a daemon or
+a multi-gigabyte model that cannot run as a documentation test:
 
 ```text
 $ agentgrep insights report --level llm --backend ollama --model llama3.2
 ```
 
+```text
+$ agentgrep insights report --level llm --backend litert-lm --model gemma-4-e2b
+```
+
+The LiteRT-LM artifact downloads on demand with `--auto-download-models`,
+or ahead of time:
+
+```text
+$ agentgrep insights models install gemma-4-e2b --level llm --backend litert-lm --yes
+```
+
 The summary is grounded in compact facts — counts, top terms, timeline,
 and open-thread titles — not raw transcripts, unless you pass
 `--include-text`.

pyproject.toml

@@ -59,6 +59,8 @@ insights-index = ["tantivy>=0.26", "sqlite-vec>=0.1.9", "numpy>=1.24"]
 insights-index-lancedb = ["lancedb>=0.33", "numpy>=1.24"]
 # Local-LLM summaries over the Ollama HTTP control plane.
 insights-llm = ["httpx>=0.28"]
+# In-process LiteRT-LM runtime for local Gemma .litertlm artifacts.
+insights-llm-litert = ["litert-lm-api>=0.13"]
 # Every stable level (1-4); excludes the -st embedding runtime and the LLM level.
 insights-all = [
   "jinja2>=3.1",

src/agentgrep/insights/enrichers/__init__.py

@@ -124,6 +124,12 @@ class EnricherContext:
             setup_command="uv pip install 'agentgrep[insights-llm]'",
             builder="build_llm",
         ),
+        BackendSpec(
+            name="litert-lm",
+            modules=("litert_lm",),
+            setup_command="uv pip install 'agentgrep[insights-llm-litert]'",
+            builder="build_llm",
+        ),
     ),
 }
 
@@ -138,6 +144,9 @@ def _ordered_backends(level: InsightsLevel, request: ReportRequest) -> tuple[Bac
     backends = _BACKENDS.get(level, ())
     if level == "index" and request.index_backend == "lancedb":
         return tuple(sorted(backends, key=lambda b: 0 if b.name == "lancedb" else 1))
+    if level == "llm" and request.llm_backend not in ("auto", ""):
+        preferred = request.llm_backend
+        return tuple(sorted(backends, key=lambda b: 0 if b.name == preferred else 1))
     return backends
 
 

src/agentgrep/insights/enrichers/llm.py

@@ -1,14 +1,16 @@
-"""Level 5 enricher: local-LLM narrative summary (Ollama over local HTTP).
-
-The summary is grounded in compact facts — counts, top terms, timeline,
-and open-thread titles — never raw transcripts unless ``--include-text``
-is set. Tokens stream to the progress sink as they arrive so the CLI can
-render the summary live. LiteRT-LM and llama.cpp remain fetch-only in
-this MVP; requesting them raises a clear configuration error.
+"""Level 5 enricher: local-LLM narrative summary.
+
+Two runtimes are wired: Ollama over local HTTP, and LiteRT-LM in-process
+(e.g. a Gemma ``.litertlm`` artifact loaded from the model cache). The
+summary is grounded in compact facts — counts, top terms, timeline, and
+open-thread titles — never raw transcripts unless ``--include-text`` is
+set. Tokens stream to the progress sink as they arrive so the CLI can
+render the summary live. llama.cpp remains fetch-only.
 """
 
 from __future__ import annotations
 
+import contextlib
 import json
 import os
 import typing as t
@@ -21,7 +23,9 @@
     from agentgrep.insights.model import InsightsReport
 
 _DEFAULT_ENDPOINT = "http://127.0.0.1:11434"
-_DEFAULT_MODEL = "llama3.2"
+_DEFAULT_OLLAMA_MODEL = "llama3.2"
+_DEFAULT_LITERT_MODEL = "gemma-4-e2b"
+_LITERT_MAX_TOKENS = 2048
 _MAX_FACT_TERMS = 12
 _MAX_FACT_THREADS = 8
 
@@ -58,48 +62,63 @@ def _build_prompt(report: InsightsReport, *, include_text: bool) -> str:
 
 
 def build_llm(ctx: EnricherContext) -> InsightsEnrichment:
-    """Stream a grounded summary from a local Ollama model."""
-    backend = ctx.request.llm_backend
-    if backend not in ("ollama", "auto"):
-        message = f"local LLM backend {backend!r} is fetch-only in this build; use --backend ollama"
-        raise BackendConfigurationError(message, level="llm")
-
+    """Stream a grounded summary from the selected local-LLM runtime."""
+    prompt = _build_prompt(ctx.report, include_text=ctx.request.include_text)
+    if ctx.backend == "litert-lm":
+        return _run_litert(ctx, prompt)
+    if ctx.backend == "ollama":
+        return _run_ollama(ctx, prompt)
+    message = f"local LLM backend {ctx.backend!r} is fetch-only in this build"
+    raise BackendConfigurationError(message, level="llm")
+
+
+def _emit_delta(ctx: EnricherContext, backend: str, model: str, accumulated: str, text: str) -> str:
+    """Emit a streamed delta to the progress sink; return new accumulated text.
+
+    Handles runtimes that yield either cumulative or incremental chunks by
+    treating a chunk that extends the accumulated text as cumulative.
+    """
+    if not text:
+        return accumulated
+    if text.startswith(accumulated):
+        delta = text[len(accumulated) :]
+        new_accumulated = text
+    else:
+        delta = text
+        new_accumulated = accumulated + text
+    if delta and ctx.progress is not None:
+        ctx.progress.llm_chunk(
+            backend=backend,
+            model=model,
+            delta=delta,
+            char_count=len(new_accumulated),
+        )
+    return new_accumulated
+
+
+def _run_ollama(ctx: EnricherContext, prompt: str) -> InsightsEnrichment:
+    """Stream a summary from a local Ollama model over HTTP."""
     httpx = ctx.modules["httpx"]
-    model = ctx.request.model or _DEFAULT_MODEL
+    model = ctx.request.model or _DEFAULT_OLLAMA_MODEL
     endpoint = _endpoint()
-    prompt = _build_prompt(ctx.report, include_text=ctx.request.include_text)
     payload = {
         "model": model,
         "messages": [{"role": "user", "content": prompt}],
         "stream": True,
     }
-
     if ctx.progress is not None:
         ctx.progress.phase("summarize", detail=f"ollama:{model}")
 
-    summary_parts: list[str] = []
+    accumulated = ""
     try:
-        with httpx.stream(
-            "POST",
-            f"{endpoint}/api/chat",
-            json=payload,
-            timeout=120.0,
-        ) as response:
+        with httpx.stream("POST", f"{endpoint}/api/chat", json=payload, timeout=120.0) as response:
             response.raise_for_status()
             for line in response.iter_lines():
                 if not line:
                     continue
                 event = json.loads(line)
                 content = event.get("message", {}).get("content", "")
-                if content:
-                    summary_parts.append(content)
-                    if ctx.progress is not None:
-                        ctx.progress.llm_chunk(
-                            backend="ollama",
-                            model=model,
-                            delta=content,
-                            char_count=sum(len(part) for part in summary_parts),
-                        )
+                accumulated = _emit_delta(ctx, "ollama", model, accumulated, accumulated + content)
                 if event.get("done"):
                     break
     except Exception as exc:
@@ -113,12 +132,93 @@ def build_llm(ctx: EnricherContext) -> InsightsEnrichment:
         failed = f"Ollama summary failed: {exc}"
         raise BackendRuntimeError(failed, level="llm") from exc
 
-    summary = "".join(summary_parts).strip()
+    return _summary_enrichment(
+        accumulated.strip(), backend="ollama", model=model, endpoint=endpoint
+    )
+
+
+def _run_litert(ctx: EnricherContext, prompt: str) -> InsightsEnrichment:
+    """Stream a summary from an in-process LiteRT-LM model artifact."""
+    from agentgrep.insights import models as models_mod
+
+    litert_lm = ctx.modules["litert_lm"]
+    model_id = ctx.request.model or _DEFAULT_LITERT_MODEL
+    spec = models_mod.resolve_llm_model(model_id, "litert-lm")
+    if spec is None or spec.artifact_filename is None:
+        message = f"no curated LiteRT-LM model {model_id!r}"
+        raise BackendConfigurationError(message, level="llm")
+
+    if not models_mod.is_installed(spec, ctx.model_cache):
+        if not ctx.policy.allow_download:
+            message = f"LiteRT-LM model {spec.model_id!r} is not provisioned"
+            install = (
+                f"agentgrep insights models install {spec.model_id} "
+                f"--level llm --backend litert-lm --yes"
+            )
+            raise BackendConfigurationError(message, level="llm", setup_command=install)
+        models_mod.install_model(
+            spec,
+            model_cache=ctx.model_cache,
+            progress=ctx.progress,
+            import_module=ctx.import_module,
+        )
+
+    model_path = models_mod.model_cache_path(spec, ctx.model_cache) / spec.artifact_filename
+    if ctx.progress is not None:
+        ctx.progress.phase("summarize", detail=f"litert-lm:{spec.model_id}")
+
+    # The LiteRT-LM C++ runtime logs model metadata to stderr at INFO; quiet it
+    # so the streamed summary is the only thing the user sees.
+    with contextlib.suppress(Exception):
+        litert_lm.set_min_log_severity(litert_lm.LogSeverity.ERROR)
+
+    accumulated = ""
+    try:
+        engine = litert_lm.Engine(
+            str(model_path),
+            backend=litert_lm.Backend.CPU,
+            max_num_tokens=_LITERT_MAX_TOKENS,
+        )
+        try:
+            conversation = engine.create_conversation()
+            for chunk in conversation.send_message_async(prompt):
+                accumulated = _emit_delta(
+                    ctx, "litert-lm", spec.model_id, accumulated, _litert_chunk_text(chunk)
+                )
+        finally:
+            engine.close()
+    except Exception as exc:
+        failed = f"LiteRT-LM summary failed: {exc}"
+        raise BackendRuntimeError(failed, level="llm") from exc
+
+    return _summary_enrichment(
+        accumulated.strip(), backend="litert-lm", model=spec.model_id, endpoint=str(model_path)
+    )
+
+
+def _litert_chunk_text(chunk: t.Any) -> str:
+    """Extract response text from a LiteRT-LM conversation chunk."""
+    content = chunk.get("content") if isinstance(chunk, dict) else None
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "".join(part.get("text", "") for part in content if isinstance(part, dict))
+    return ""
+
+
+def _summary_enrichment(
+    summary: str,
+    *,
+    backend: str,
+    model: str,
+    endpoint: str,
+) -> InsightsEnrichment:
+    """Build the enrichment payload for a generated summary."""
     return InsightsEnrichment(
         level="llm",
-        backend="ollama",
+        backend=backend,
         status="ok",
-        message=f"summarized via ollama:{model}",
+        message=f"summarized via {backend}:{model}",
         data={"summary": summary, "model": model, "endpoint": endpoint},
-        provenance={"backend": "ollama", "model": model, "endpoint": endpoint},
+        provenance={"backend": backend, "model": model, "endpoint": endpoint},
     )

src/agentgrep/insights/models.py

@@ -131,7 +131,7 @@ def files(self) -> tuple[str, ...]:
         license="Gemma",
         source_url="https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm",
         local_id="gemma4-e2b",
-        notes="LiteRT-LM Gemma 4 E2B; fetch-only registry parity in this MVP.",
+        notes="LiteRT-LM Gemma 4 E2B; runs in-process via agentgrep[insights-llm-litert].",
     ),
     LLMModelSpec(
         model_id="phi-4-mini-gguf",

tests/test_import_time.py

@@ -36,6 +36,7 @@
     "httpx",
     "jinja2",
     "huggingface_hub",
+    "litert_lm",
 )
 
 

tests/test_insights_enrichers.py

@@ -336,3 +336,66 @@ def test_llm_enricher_streams_grounded_summary() -> None:
     assert enrichment.provenance is not None
     assert enrichment.provenance["backend"] == "ollama"
     assert progress.deltas == ["Worked on ", "indexing."]
+
+
+def _fake_litert_lm() -> t.Any:
+    """Return a fake litert_lm whose conversation yields cumulative chunks."""
+
+    class _Conversation:
+        def send_message_async(self, _prompt: str) -> t.Iterator[dict[str, str]]:
+            for text in ("Worked on ", "Worked on indexing."):
+                yield {"role": "model", "content": text}
+
+    class _Engine:
+        def __init__(self, _path: str, **_kwargs: object) -> None:
+            pass
+
+        def create_conversation(self) -> _Conversation:
+            return _Conversation()
+
+        def close(self) -> None:
+            pass
+
+    return types.SimpleNamespace(
+        Engine=_Engine,
+        Backend=types.SimpleNamespace(CPU=object()),
+        LogSeverity=types.SimpleNamespace(ERROR=object()),
+        set_min_log_severity=lambda _severity: None,
+    )
+
+
+def test_llm_enricher_litert_streams_from_local_model(tmp_path: pathlib.Path) -> None:
+    """The litert-lm backend loads a provisioned model and streams a summary."""
+    spec = models_mod.resolve_llm_model("gemma-4-e2b", "litert-lm")
+    assert spec is not None
+    target = models_mod.model_cache_path(spec, tmp_path)
+    target.mkdir(parents=True, exist_ok=True)
+    (target / "agentgrep-manifest.json").write_text("{}", encoding="utf-8")
+
+    progress = _RecordingProgress()
+    report = build_report(
+        _RECORDS,
+        ReportRequest(requested_level="llm", llm_backend="litert-lm", model="gemma-4-e2b"),
+        import_module=_importer({"litert_lm": _fake_litert_lm()}),
+        progress=progress,
+        model_cache=tmp_path,
+    )
+    enrichment = report.enrichments[0]
+    assert enrichment.status == "ok"
+    assert enrichment.backend == "litert-lm"
+    assert enrichment.data["summary"] == "Worked on indexing."
+    assert progress.deltas == ["Worked on ", "indexing."]
+
+
+def test_llm_enricher_litert_errors_when_model_not_provisioned(tmp_path: pathlib.Path) -> None:
+    """An unprovisioned litert model yields an error enrichment with an install hint."""
+    report = build_report(
+        _RECORDS,
+        ReportRequest(requested_level="llm", llm_backend="litert-lm", model="gemma-4-e2b"),
+        import_module=_importer({"litert_lm": _fake_litert_lm()}),
+        model_cache=tmp_path,
+    )
+    enrichment = report.enrichments[0]
+    assert enrichment.status == "error"
+    setup = next(d.setup_command for d in report.diagnostics if d.setup_command)
+    assert "models install" in setup