feat(kg-rag): KG-RAG semantic-graph layer on the new kb-server architecture

.env.example

@@ -1,6 +1,79 @@
+# ============================================================================
+# LAMB root Docker Compose environment (docker-compose.yaml)
+# ============================================================================
+# Copy this file to `.env` in the repository root (next to docker-compose.yaml).
+# Docker Compose loads it automatically. Per-service application config still
+# lives in backend/.env, lamb-kb-server/backend/.env, etc.
+
+# Absolute path to this checkout on the host. REQUIRED — the dev compose
+# bind-mounts it into every container, so it must be correct.
 LAMB_PROJECT_PATH=/opt/lamb
+
+# Open WebUI integration
 OWI_BASE_URL=http://openwebui:8080
 OWI_PUBLIC_BASE_URL=http://localhost:8080
 OWI_PATH=/opt/lamb/open-webui/backend/data
+
+# LAMB data + legacy KB server (port 9090)
 LAMB_DB_PATH=/opt/lamb
-LAMB_KB_SERVER=http://kb:9090
+LAMB_KB_SERVER=http://kb:9090
+
+# New KB server (Knowledge Stores, port 9092) — backs the KG-RAG feature.
+# The backend reaches it at these values; the token must match the
+# LAMB_API_TOKEN the kb-v2 service starts with (compose default: 0p3n-w3bu!).
+LAMB_KB_SERVER_V2=http://kb-v2:9092
+LAMB_KB_SERVER_V2_TOKEN=0p3n-w3bu!
+
+# Library Manager (port 9091) bearer token — must match on backend + service.
+LIBRARY_MANAGER_TOKEN=change-me
+
+# ============================================================================
+# KG-RAG / Semantic Graph (optional feature on this branch)
+# ============================================================================
+# KG-RAG adds a Neo4j-backed semantic graph on top of vector retrieval in the
+# new KB server (kb-v2). It is OFF by default and fully isolated: with
+# KG_RAG_ENABLED=false the /graph + /benchmarks endpoints are never mounted and
+# every existing simple / hierarchical RAG flow behaves exactly as before.
+#
+# Neo4j only starts under the `kg-rag` compose profile, so the base stack stays
+# unchanged unless you explicitly opt in:
+#
+#   docker compose --profile kg-rag up -d
+#
+# See Documentation/kg-rag-deployment.md for the full walkthrough.
+
+# Master switch. Set to true (and use --profile kg-rag) to enable KG-RAG.
+KG_RAG_ENABLED=false
+
+# Run LLM concept extraction + a Neo4j write on every ingestion job. When true,
+# single-document ingestion grows from sub-second to several seconds. Set false
+# to keep ingestion fast and back-populate the graph later via the migrate API.
+KG_RAG_INDEX_ON_INGEST=false
+
+# OpenAI key used for concept/entity extraction. Per-request keys (threaded
+# from the LAMB org config) are preferred; this is the ingestion-time fallback.
+KG_RAG_OPENAI_API_KEY=
+
+# Models used for graph reasoning and concept extraction.
+KG_RAG_CHAT_MODEL=gpt-4o-mini
+# Leave empty to fall back to KG_RAG_CHAT_MODEL / gpt-4o-mini.
+KG_RAG_EXTRACTION_MODEL=
+
+# Neo4j connection. The neo4j service is created from these values too, so the
+# password set here is the password the database is initialised with.
+KG_RAG_NEO4J_URI=bolt://neo4j:7687
+KG_RAG_NEO4J_USER=neo4j
+KG_RAG_NEO4J_PASSWORD=lamb-kg-rag-password
+
+# Graph traversal tuning (clamped server-side: depth 1-4, limit_factor 1-20).
+KG_RAG_GRAPH_DEPTH=2
+KG_RAG_LIMIT_FACTOR=4
+KG_RAG_EXTRACTION_MAX_WORKERS=4
+
+# kb-v2 query-plugin governance (ENABLE | ADVANCED | DISABLE).
+PLUGIN_KG_RAG_QUERY=ADVANCED
+
+# Neo4j JVM sizing (optional — raise for larger graphs).
+# NEO4J_HEAP_INITIAL_SIZE=512m
+# NEO4J_HEAP_MAX_SIZE=1G
+# NEO4J_PAGECACHE_SIZE=512m

.env.next.example

@@ -110,3 +110,41 @@ OWI_ADMIN_PASSWORD=CHANGE_ME_strong_admin_password
 # CADDY_EMAIL=admin@yourdomain.com
 # LAMB_PUBLIC_HOST=lamb.yourdomain.com
 # OWI_PUBLIC_HOST=owi.lamb.yourdomain.com
+
+# ============================================================================
+# KG-RAG / Semantic Graph (optional)
+# ============================================================================
+# Enable Graph RAG / KG-RAG in the new KB server. Requires the ``kg-rag``
+# Compose profile (starts Neo4j):
+#   docker compose -f docker-compose.next.yaml --profile kg-rag up -d
+#
+# When KG_RAG_ENABLED=false (default), graph endpoints are not registered
+# and existing simple / hierarchical / parent-child RAG flows are unchanged.
+#
+# Per-collection opt-in: even with the flag on, a Knowledge Store only
+# uses graph features when its ``graph_enabled`` field is true (set in
+# create/migrate flow).
+#
+# OpenAI credentials for concept extraction can be sent per-request
+# (preferred) via the X-OpenAI-Api-Key header on /graph endpoints. The
+# server-level KG_RAG_OPENAI_API_KEY is a fallback used during ingestion
+# when no per-request key has been threaded through.
+
+# KG_RAG_ENABLED=false
+# Note on KG_RAG_INDEX_ON_INGEST: when enabled, every ingestion job ALSO
+# runs LLM concept extraction (1 OpenAI call per parent text) and a Neo4j
+# write. Expect single-document ingestion latency to grow from sub-second
+# to multiple seconds. Set to false to keep ingestion fast and run
+# `/graph/.../migrate` manually instead.
+# KG_RAG_INDEX_ON_INGEST=true
+# KG_RAG_OPENAI_API_KEY=
+# KG_RAG_CHAT_MODEL=gpt-4o-mini
+# KG_RAG_EXTRACTION_MODEL=gpt-5-nano
+# KG_RAG_NEO4J_URI=bolt://neo4j:7687
+# KG_RAG_NEO4J_USER=neo4j
+# KG_RAG_NEO4J_PASSWORD=change-this-password
+# KG_RAG_GRAPH_DEPTH=2
+# KG_RAG_LIMIT_FACTOR=4
+# KG_RAG_EXTRACTION_MAX_WORKERS=4
+# Per-request OpenAI timeout for extraction calls. Default 60s.
+# KG_RAG_OPENAI_TIMEOUT_SECONDS=60

Documentation/kg-rag-deployment.md

@@ -0,0 +1,157 @@
+# KG-RAG / Semantic-Graph Deployment Guide
+
+This guide explains how to bring up the LAMB stack with the **KG-RAG (Knowledge-Graph RAG)** feature enabled — the Neo4j-backed semantic-graph layer that augments vector retrieval in the new KB server.
+
+KG-RAG is **optional and off by default**. With `KG_RAG_ENABLED=false` the graph endpoints are never mounted and every existing RAG flow (simple, hierarchical, single-file, context-aware) behaves exactly as before. You only get the extra services and behaviour when you explicitly opt in as described below.
+
+> Scope: this targets the development `docker-compose.yaml` at the repository root, which is the compose file that wires the new KB server (`kb-v2`, port 9092) and Neo4j. It bind-mounts the checkout into every container and installs dependencies on first start.
+
+---
+
+## What KG-RAG adds to the stack
+
+| Service | Port | Role with KG-RAG | Started by |
+|---|---|---|---|
+| `kb-v2` (new KB server) | 9092 | Concept extraction, graph write, graph-augmented query | always (base stack) |
+| `neo4j` | 7474 (HTTP), 7687 (Bolt) | Semantic graph + curation audit log | **only `--profile kg-rag`** |
+
+`kb-v2` always runs, but it only mounts the `/graph` + `/benchmarks` routers and talks to Neo4j when `KG_RAG_ENABLED=true`. Neo4j itself sits behind the `kg-rag` Compose profile, so the base `docker compose up` never starts it.
+
+---
+
+## Prerequisites
+
+- Docker Engine 24+ and the Docker Compose v2 plugin (`docker compose version`).
+- This repository checked out on the **`feat/kg-rag-new-kbserver-arch`** branch.
+- An OpenAI-compatible API key if you want concept extraction to actually run (extraction is an LLM call). Without a key the stack still starts; ingestion just skips graph writes / logs an extraction error.
+- ~2 GB free RAM for Neo4j (default heap 1 GB + page cache).
+
+---
+
+## Step 1 — Create the root `.env`
+
+From the repository root:
+
+```bash
+cp .env.example .env
+```
+
+Edit `.env` and set, at minimum:
+
+```bash
+# REQUIRED — absolute path to THIS checkout on the host
+LAMB_PROJECT_PATH=/absolute/path/to/lamb
+
+# Turn the feature on
+KG_RAG_ENABLED=true
+
+# Pick a Neo4j password (this initialises the database on first boot)
+KG_RAG_NEO4J_PASSWORD=lamb-kg-rag-password
+
+# Key used for concept extraction at ingest time (optional but recommended)
+KG_RAG_OPENAI_API_KEY=sk-...
+```
+
+Everything else has working defaults (see the comments in `.env.example`). The KG-RAG block controls graph behaviour:
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `KG_RAG_ENABLED` | `false` | Master switch. Must be `true` for any graph behaviour. |
+| `KG_RAG_INDEX_ON_INGEST` | `false` | If `true`, run extraction + graph write on every ingestion job (slower). If `false`, ingest fast and back-populate later via the migrate API. |
+| `KG_RAG_OPENAI_API_KEY` | empty | Fallback key for extraction (per-request keys from LAMB are preferred). |
+| `KG_RAG_CHAT_MODEL` | `gpt-4o-mini` | Model for graph reasoning. |
+| `KG_RAG_EXTRACTION_MODEL` | empty → chat model | Model for concept/entity extraction. |
+| `KG_RAG_NEO4J_URI` | `bolt://neo4j:7687` | Neo4j Bolt URI (service name inside the compose network). |
+| `KG_RAG_NEO4J_USER` / `KG_RAG_NEO4J_PASSWORD` | `neo4j` / `lamb-kg-rag-password` | Neo4j credentials; the DB is initialised with these. |
+| `KG_RAG_GRAPH_DEPTH` | `2` | Graph traversal depth (clamped 1–4). `.env.example` ships `2`; note the kb-v2 service's bare compose fallback is `1`, so keep this line set rather than deleting it. |
+| `KG_RAG_LIMIT_FACTOR` | `4` | Candidate-expansion factor (clamped 1–20). |
+
+The same file also sets `LAMB_KB_SERVER_V2_TOKEN` and `LIBRARY_MANAGER_TOKEN`; leave the defaults unless you change them on the services too.
+
+## Step 2 — Create `backend/.env`
+
+The backend container loads `backend/.env` directly (`env_file:`), so it must exist. From the **repository root**:
+
+```bash
+cp backend/.env.example backend/.env
+```
+
+Set a real `OPENAI_API_KEY` (and any other provider keys) inside `backend/.env`. No KG-RAG-specific edits are needed here — the backend learns the new KB server address from `LAMB_KB_SERVER_V2` / `LAMB_KB_SERVER_V2_TOKEN`, which the compose file injects from the root `.env`.
+
+> You do **not** need to create a `.env` for the KB servers by hand. `kb-v2` (the KG-RAG server) takes all of its config from the compose `environment:` block — it never reads a `.env` file. The legacy `kb` service auto-creates `lamb-kb-server-stable/backend/.env` from its `.env.example` on first start. Either way, the KG-RAG settings come from compose environment variables sourced from the root `.env`.
+
+## Step 3 — Start the stack with the `kg-rag` profile
+
+```bash
+docker compose --profile kg-rag up -d
+```
+
+The `--profile kg-rag` flag is what adds the `neo4j` service. Without it, `kb-v2` starts but has no graph database to talk to.
+
+> This assumes a clean host: ports 9099, 9092, 9090, 9091, 8080, 5173, 7474 and 7687 must be free. If a LAMB stack is already running, `up` will fail on a port conflict — bring the existing one down first (`docker compose --profile kg-rag down`) or start the new one with a distinct `COMPOSE_PROJECT_NAME` and remapped ports.
+
+First boot is slow: the containers `pip install` / `npm build` against the bind-mounted source. Watch progress with:
+
+```bash
+docker compose --profile kg-rag ps
+docker compose logs -f kb-v2
+```
+
+You can also bring up just the feature-critical services to validate KG-RAG quickly without building the frontend/OpenWebUI:
+
+```bash
+docker compose --profile kg-rag up -d neo4j kb-v2
+```
+
+## Step 4 — Verify KG-RAG is live
+
+1. **Neo4j is healthy:**
+   ```bash
+   docker compose ps neo4j     # STATUS shows "healthy" (ps lists running containers; no profile flag needed)
+   ```
+   The Neo4j browser is at <http://localhost:7474> (log in with `KG_RAG_NEO4J_USER` / `KG_RAG_NEO4J_PASSWORD`).
+
+2. **kb-v2 mounted the graph routers** — the startup log prints this line only when the feature is on:
+   ```bash
+   docker compose logs kb-v2 | grep "KG-RAG enabled"
+   # → KG-RAG enabled: mounted /graph and /benchmarks routers
+   ```
+
+3. **kb-v2 is healthy and the graph surface exists:**
+   ```bash
+   curl -s http://localhost:9092/health        # {"status":"ok",...}
+   curl -s http://localhost:9092/openapi.json | grep -o '"/graph[^"]*"' | head
+   # → graph paths appear ONLY when KG_RAG_ENABLED=true
+   ```
+   With the feature off, `/graph` paths are absent from the OpenAPI spec (callers get a clean 404).
+
+## Step 5 — Use it from LAMB
+
+1. Open the frontend at <http://localhost:9099> (or the Svelte dev server at <http://localhost:5173>).
+2. Create a **Knowledge Store** via the "Create Knowledge" wizard. In the Setup step, enable the **graph / `graph_enabled`** toggle and pick the extraction vendor/model. This per-store opt-in is required *in addition* to `KG_RAG_ENABLED` — the flag enables the capability, the toggle enables it for that store. Store setup is locked after creation.
+3. Link Library items into the store to ingest them. If `KG_RAG_INDEX_ON_INGEST=true`, concepts are extracted and written to Neo4j during ingestion; otherwise run the migrate/back-populate action from the store's graph view.
+4. Inspect and curate the graph in the store's **Graph** view (Sigma.js): approve/reject/rename/merge concepts; every edit is recorded as an audit `ChangeEvent`.
+5. Attach the store to an assistant whose RAG processor is `knowledge_store_rag` to use graph-augmented retrieval at query time.
+
+## Turning KG-RAG off again
+
+```bash
+docker compose --profile kg-rag down       # stop everything incl. Neo4j
+```
+
+Set `KG_RAG_ENABLED=false` in `.env` and start normally (`docker compose up -d`, no profile). The graph routers disappear, Neo4j is not started, and all vector RAG flows continue unchanged. Neo4j data persists in the `neo4j-data` volume for next time.
+
+---
+
+## Troubleshooting
+
+| Symptom | Cause / fix |
+|---|---|
+| `kb-v2` log has no "KG-RAG enabled" line | `KG_RAG_ENABLED` not `true` in `.env`, or container not recreated — run `docker compose up -d --force-recreate kb-v2`. |
+| `/graph` calls return 404 | Feature flag off, or you hit the legacy `kb` (9090) instead of `kb-v2` (9092). |
+| kb-v2 logs Neo4j connection errors | Neo4j not started (missing `--profile kg-rag`), wrong `KG_RAG_NEO4J_URI`, or password mismatch. Confirm `docker compose ps neo4j` is healthy. |
+| Ingestion is suddenly slow | `KG_RAG_INDEX_ON_INGEST=true` runs an LLM call per parent text. Set it `false` and back-populate via the migrate API. |
+| Extraction silently does nothing | No OpenAI key reached the server — set `KG_RAG_OPENAI_API_KEY` or thread a per-request key from the LAMB org config. |
+| `.env` changes not picked up | Compose only re-reads env on container (re)create: `docker compose up -d --force-recreate <service>` (a plain `restart` is not enough). |
+
+See `lamb-kb-server/Documentation/` for the architecture decision records (ADR-KS-1…17) behind this design.

README.md

@@ -89,6 +89,12 @@ For development or custom deployments:
 
 📘 **[Complete Installation Guide](Documentation/installationguide.md)** - Step-by-step manual setup for all components
 
+### Optional: KG-RAG (Semantic Graph)
+
+To bring up the stack with the Neo4j-backed Knowledge-Graph RAG feature enabled:
+
+📘 **[KG-RAG Deployment Guide](Documentation/kg-rag-deployment.md)** - Enable graph-augmented retrieval with the `kg-rag` Compose profile
+
 ### Quick Overview
 
 LAMB requires four main services:

backend/.coveragerc

@@ -0,0 +1,6 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING:
+    raise NotImplementedError
+    if __name__ == .__main__.:

backend/creator_interface/kb_server_manager.py

@@ -23,7 +23,13 @@
 
 # Get environment variables
 _raw_kb_server = os.getenv('LAMB_KB_SERVER', None)
-LAMB_KB_SERVER = _raw_kb_server or 'http://172.17.0.1:9090'
+# Redirect table for dev environments where the configured URL is unreachable.
+# In the normal docker-compose topology, `kb` resolves correctly inside the
+# `lamb-*` bridge network, so the table is empty. The runtime fallback in
+# is_kb_server_available() still tries host.docker.internal as a backup when
+# the primary `http://kb:9090` is unreachable.
+_KB_REDIRECTS: dict[str, str] = {}
+LAMB_KB_SERVER = _KB_REDIRECTS.get(_raw_kb_server, _raw_kb_server) or 'http://172.17.0.1:9090'
 LAMB_KB_SERVER_TOKEN = os.getenv('LAMB_KB_SERVER_TOKEN')
 if not LAMB_KB_SERVER_TOKEN:
     raise ValueError("LAMB_KB_SERVER_TOKEN environment variable is required")
@@ -75,8 +81,9 @@ def _get_kb_config_for_user(self, creator_user: Dict[str, Any]) -> Dict[str, Any
                 api_token = kb_config.get('api_token')
                 if not api_token:
                     api_token = self.global_kb_server_token
+                org_url = kb_config.get('server_url')
                 result = {
-                    'url': kb_config.get('server_url'),
+                    'url': _KB_REDIRECTS.get(org_url, org_url),
                     'token': api_token
                 }
                 # Pass through embedding model and collection defaults if configured
@@ -97,8 +104,9 @@ def _get_kb_config_for_user(self, creator_user: Dict[str, Any]) -> Dict[str, Any
         # Fallback to global environment variables
         if not self.global_kb_server_url:
             raise ValueError("LAMB_KB_SERVER environment variable is required")
+        resolved_url = _KB_REDIRECTS.get(self.global_kb_server_url, self.global_kb_server_url)
         return {
-            'url': self.global_kb_server_url,
+            'url': resolved_url,
             'token': self.global_kb_server_token
         }
 
@@ -501,8 +509,12 @@ async def create_knowledge_base(self, kb_data: KnowledgeBaseCreate, creator_user
 
         # Update kb_data with sanitized name
         kb_data.name = sanitized_name
-   
+
         # Create collection in KB server
+        # Apply host redirect for dev/test environments where 'kb' DNS doesn't resolve
+        _alt_url = _KB_REDIRECTS.get(kb_server_url)
+        if _alt_url:
+            kb_server_url = _alt_url
         async with httpx.AsyncClient() as client:
             kb_server_collections_url = f"{kb_server_url}/collections"
             logger.info(f"Creating collection in KB server at {kb_server_collections_url}: {sanitized_name}")