feat(core): diversify retrieval results by document (#161)

Release highlights:
- Broaden top-k document coverage while preserving the requested result count.
- Keep lexical fallback available for corpora larger than one scan batch.

Release details:
- Add a configurable document cap across CLI, MCP, ask, evaluation, and portable status.
- Over-retrieve before ranking, then diversify with ranked backfill.
- Scan lexical fallback batches across the full eligible corpus.
- Refresh compatible dependencies and pin patched transitive packages.

Verification:
- Pass pnpm validate on the release branch with 609 tests.
- Pass audit, CI, Linux, macOS, Commitlint, and CodeQL checks.
- Pass 100/100 reference quality cases with recall@10 1.000.

Author: jb-thery

README.md

@@ -224,7 +224,10 @@ pnpm exec rgr research "release obligations" --compact --timeout-ms 10000
 ```
 
 `preview` inspects redacted chunks without writing an index. `audit` compares sources with indexed
-state. `research` combines bounded query variants with deterministic cross-query ranking. Use
+state. Search keeps one primary passage per document by default, over-retrieves before applying the
+cap, and preserves ranked backfill when fewer distinct documents are available. Override the cap
+with `--max-chunks-per-document`; context-radius neighbors remain attached to their primary result.
+`research` combines bounded query variants with deterministic cross-query ranking. Use
 `rgr doctor --deep` only when you need a live O(corpus) inventory; normal status and doctor checks
 read the compact activation manifest.
 

biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/2.5.1/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.5.9/schema.json",
   "vcs": {
     "enabled": true,
     "clientKind": "git",

docs/api-reference.md

@@ -58,13 +58,13 @@ instead of presenting an unverifiable line claim.
 
 With `explain: true`, `score` includes the vector and lexical ranks, their reciprocal-rank-fusion
 contributions, matched terms, backend scores, FTS or complete-fallback activation and reason,
-candidate materialization, query-variant count, indexed/unindexed rows, coverage, queue wait as
-`workloadQueueMs`, and
-`rankingPolicyFingerprint`. The fingerprint
-identifies the provider, retrieval profile, fusion parameters, and abstention threshold used by
-the result. Equal scores are ordered by stable source and chunk keys, so identical indexes return
-the same order regardless of backend row order. Search returns an empty array when every candidate
-fails the active provider's evidence threshold.
+fallback scan batches, candidate materialization, query-variant count, indexed/unindexed rows,
+coverage, document-cap and ranked-backfill state, queue wait as `workloadQueueMs`, and
+`rankingPolicyFingerprint`. The fingerprint identifies the provider, retrieval profile, document
+cap, fusion parameters, and abstention threshold used by the result. Equal scores are ordered by
+stable source and chunk keys, so identical indexes return the same order regardless of backend row
+order. Search returns an empty array when every candidate fails the active provider's evidence
+threshold.
 
 ### Persistent client for Node.js workers
 
@@ -236,13 +236,17 @@ to perform an external action.
 
 One evaluation pins a single configuration, connection, manifest generation, table handle, and
 embedding model. Cases run with bounded concurrency, preserve file order in the report, and release
-all scoped resources when evaluation finishes.
+all scoped resources when evaluation finishes. The report records the configured
+`maxChunksPerDocument` beside the ranking-policy fingerprint so reference results are reproducible.
 
-`SearchOptions` accepts `cwd`, `topK`, `contextRadius`, `includePaths`, `excludePaths`,
-`contextPaths`, `explain`, `vectorSearchMode`, `signal`, and `timeoutMs`. Set
+`SearchOptions` accepts `cwd`, `topK`, `maxChunksPerDocument`, `contextRadius`, `includePaths`,
+`excludePaths`, `contextPaths`, `explain`, `vectorSearchMode`, `signal`, and `timeoutMs`. Set
 `vectorSearchMode: "exact"` to bypass ANN for diagnostic comparison; the default `"adaptive"`
-uses the compatible strategy recorded in the manifest. `topK` is limited to 100 and
-`contextRadius` is clamped to three chunks. `IngestOptions` also accepts `rebuild`, a
+uses the compatible strategy recorded in the manifest. `topK` and `maxChunksPerDocument` are
+limited to 100. The document cap defaults to one, applies after scoring, and is preceded by internal
+over-retrieval. Ranked backfill preserves the requested result count when too few distinct
+documents are available. `contextRadius` is clamped to three chunks and attaches neighbors after
+primary-result diversification. `IngestOptions` also accepts `rebuild`, a
 positive `batchSize` that defaults to 25 files and is capped at 128, `incrementalFailurePolicy`, and
 an optional `onProgress` callback. Set `collectMetrics: true` to include privacy-safe phase,
 throughput, cache-state, RSS, OCR subprocess, fallback, error, timeout, and bound-activation metrics

docs/cli-reference.md

@@ -40,25 +40,29 @@ rgr sources add "docs/**/*.md" "!docs/archive/**"
 rgr sources list
 rgr preview --path docs --max-files 5 --max-chunks 3
 rgr search "migration" --top-k 5 --context-radius 1
+rgr search "migration" --top-k 5 --max-chunks-per-document 2
 rgr search "migration" --include-path docs --exclude-path docs/archive
 rgr search "migration" --context-path "Guide > Migration" --explain
 rgr search "migration" --exact-vector-search
 ```
 
 `sources add` accepts paths, globs, and `!` exclusions. Search, ask, and research accept `--top-k`,
 `--include-path`, `--exclude-path`, and repeatable `--context-path`. Search and ask accept
-`--explain`; the optional score object reports RRF contributions, retriever ranks, raw backend
-scores, FTS or complete-fallback activation and reason, candidate and index coverage, queue wait,
-and matched query terms without changing ranking. Use `--compact` on search or research when
+`--max-chunks-per-document` and `--explain`. The document cap defaults to one, applies after scoring,
+and over-retrieves internally before final truncation. Ranked backfill keeps the requested result
+count when the corpus has too few distinct documents. The optional score object reports RRF
+contributions, retriever ranks, raw backend scores, document-cap and backfill state, FTS or
+complete-fallback activation and reason, fallback scan batches, candidate and index coverage, queue
+wait, and matched query terms without changing ranking. Use `--compact` on search or research when
 agent context is limited. This remains explicit for CLI automation; MCP search, ask, and research
-are compact by default. Search and ask accept `--exact-vector-search` to bypass an active ANN
-index for diagnostics against exhaustive vector search. `--top-k` is limited to 100 and
-`--context-radius` is clamped to three chunks.
+are compact by default. Search and ask accept `--exact-vector-search` to bypass an active ANN index
+for diagnostics against exhaustive vector search. `--top-k` and `--max-chunks-per-document` are
+limited to 100, and `--context-radius` is clamped to three chunks.
 
 The explanation also contains a ranking-policy fingerprint so a stored quality report can be tied
-to the exact provider, profile, fusion, and abstention settings. Equal backend scores have a stable
-source-and-chunk tie-break. Search returns no result when all candidates fail the provider-aware
-evidence threshold; it does not force a low-confidence passage into the response.
+to the exact provider, profile, document cap, fusion, and abstention settings. Equal backend scores
+have a stable source-and-chunk tie-break. Search returns no result when all candidates fail the
+provider-aware evidence threshold; it does not force a low-confidence passage into the response.
 
 `preview` uses the active redaction and chunking configuration but never writes storage. `audit`
 reports min, mean, p50, p95, and max chunk sizes plus structural-context coverage.

docs/configuration.md

@@ -24,6 +24,7 @@ edit JSON only for a real need.
 | `embeddingModelRevision` | Pinned commit for bundled profiles | Use an immutable 40-character commit for reproducible model artifacts. Unknown custom models default to the mutable `main` revision until explicitly pinned. |
 | `embeddingModelDigest` | `null` | `rgr models pull --enable` records a SHA-256 identity for the resolved local artifact tree. Do not set it by hand unless the local files were verified independently. |
 | `topK` | `8` | Change the CLI and TypeScript default, up to the hard limit of 100. MCP requests without `topK` start at the lower of this value and 3. |
+| `maxChunksPerDocument` | `1` | Limit primary ranked passages from one document before final `topK` truncation, up to 100. Ranked backfill preserves the requested result count when the candidate set has too few documents. |
 | `mcpMaxTopK` | `10` | Bound explicit MCP passage requests; values above 100 are rejected. |
 | `mcpMaxOutputBytes` | `32768` | Cap variable-size MCP tool and resource JSON; the server also enforces an absolute 1 MiB ceiling. |
 | `chunkSize` / `chunkOverlap` | `1200` / `200` | Tune chunking, then rebuild the index. Chunk size is capped at 1,000,000 characters. |
@@ -32,7 +33,7 @@ edit JSON only for a real need.
 | `embeddingBatchSize` | `32` | Bound one model call; values above `128` are rejected. |
 | `sourceFingerprintMode` | `fast` | Use `strict` to hash every source on every inventory instead of reusing unchanged private fingerprints. |
 | `incrementalFailurePolicy` | `preserve-last-good` | Use `remove-stale` only when failed changed files must disappear immediately. |
-| `hybridTextScanLimit` | `5000` | Bound only the complete-scan fallback used when FTS is unavailable; values above 10,000 are rejected. A fallback smaller than the active corpus is rejected instead of returning silently truncated lexical evidence. |
+| `hybridTextScanLimit` | `5000` | Set the batch size for the complete lexical scan used when FTS is unavailable; values above 10,000 are rejected. The fallback scans every matching chunk across as many batches as the corpus requires. |
 | `workloadLimits` | See below | Bound active search, embedding, and ingestion work plus their queues and queue deadlines. |
 | `includeExtensions` | `[]` | Add safe custom text extensions. |
 
@@ -81,21 +82,33 @@ Profiles bound retrieval work. They are candidate and diversification budgets, n
 a larger budget improves every corpus. Evaluate the profile against a representative golden set
 before changing production configuration.
 
-| Profile | Quality intent | Latency intent | Default `topK` | Fallback scan cap | Vector candidates | FTS candidates | First-pass chunks per source | Context radius |
+| Profile | Quality intent | Latency intent | Default `topK` | Fallback batch size | Vector candidates | FTS candidates | Document cap | Context radius |
 | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| `fast` | Narrow, diverse evidence | Lowest work budget | 5 | 2,000 | max(40, 3 x `topK`) | max(100, 10 x `topK`) | 1, then backfill | 0 |
-| `balanced` | General-purpose evidence | Default work budget | 8 | 5,000 | max(80, 4 x `topK`) | max(250, 20 x `topK`) | 2, then backfill | 0 |
-| `quality` | Broader multi-section evidence | Highest work budget | 12 | 10,000 | max(200, 8 x `topK`) | min(4,000, max(500, 40 x `topK`)) | 4, then backfill | 1 |
-| `custom` | Golden-set validated | Operator-defined | configured | configured | max(80, 4 x `topK`) | max(250, 20 x `topK`) | 2, then backfill | 0 |
+| `fast` | Narrow, diverse evidence | Lowest work budget | 5 | 2,000 | max(40, 3 x `demand`) | max(100, 10 x `demand`) | configured, default 1 | 0 |
+| `balanced` | General-purpose evidence | Default work budget | 8 | 5,000 | max(80, 4 x `demand`) | max(250, 20 x `demand`) | configured, default 1 | 0 |
+| `quality` | Broader multi-section evidence | Highest work budget | 12 | 10,000 | max(200, 8 x `demand`) | min(4,000, max(500, 40 x `demand`)) | configured, default 1 | 1 |
+| `custom` | Golden-set validated | Operator-defined | configured | configured | max(80, 4 x `demand`) | max(250, 20 x `demand`) | configured, default 1 | 0 |
 
+`demand` is `topK * ceil(4 / maxChunksPerDocument)`, with a minimum multiplier of one. This internal
+over-retrieval gives the diversity pass enough lower-ranked documents before final truncation.
 Vector candidates are capped at 1,000. The FTS pool is profile-aware and capped at 4,000,
-independently from `hybridTextScanLimit`. Structural context and body text feed the primary local
-index. Exact file paths use a bounded scalar variant. Controlled exact-phrase, identifier, and fuzzy
-rare-term queries expand only a primary pool that cannot fill `topK`, preserving established ranks.
-The diversity pass prefers distinct sources first, then backfills ranked non-duplicate, non-overlapping
-chunks to `topK`. Hybrid ranking uses deterministic reciprocal-rank fusion with `k = 60` and equal
-vector and lexical weights. Stable source and chunk keys break score ties before ranks are assigned.
-The active provider, profile, and ranking parameters form a policy fingerprint stored in quality
+independently from the complete-scan batch size. Structural context and body text feed the primary
+local index. Exact file paths use a bounded scalar variant. Controlled exact-phrase, identifier,
+and fuzzy rare-term queries expand only a primary pool that cannot fill the demand, preserving
+established ranks.
+
+After scoring and abstention, the deterministic diversity pass keeps at most
+`maxChunksPerDocument` primary passages per relative path while preserving rank order. It then
+backfills ranked, non-duplicate, non-overlapping chunks only when too few documents or candidates
+remain to fill `topK`. Neighbor chunks requested through `contextRadius` are attached afterward and
+do not consume the primary document cap. MMR is not part of the default policy: pairwise similarity
+adds more work and corpus-dependent ordering, and the current golden benchmark does not show an
+advantage over the simple cap default. It can be evaluated later as an explicit opt-in strategy
+without changing this predictable default.
+
+Hybrid ranking uses deterministic reciprocal-rank fusion with `k = 60` and equal vector and lexical
+weights. Stable source and chunk keys break score ties before ranks are assigned. The active
+provider, profile, document cap, and ranking parameters form a policy fingerprint stored in quality
 reports and exposed by score explanations.
 
 Abstention is provider-aware. `local-hash` requires lexical evidence and gives query identifiers

package.json

@@ -74,12 +74,12 @@
     "validate": "pnpm lint && pnpm audit:security && pnpm check && pnpm test:coverage && pnpm build && pnpm api:check && pnpm smoke && pnpm package:check && pnpm release:semantic:smoke && pnpm release:artifacts"
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.5.1",
-    "@commitlint/cli": "^21.1.0",
-    "@commitlint/config-conventional": "^21.1.0",
+    "@biomejs/biome": "^2.5.9",
+    "@commitlint/cli": "^21.2.2",
+    "@commitlint/config-conventional": "^21.2.2",
     "@semantic-release/exec": "^7.1.0",
-    "@vitest/coverage-v8": "4.1.9",
-    "semantic-release": "^25.0.5",
+    "@vitest/coverage-v8": "4.1.10",
+    "semantic-release": "^25.0.9",
     "yaml": "^2.9.0"
   },
   "main": "index.js",

packages/ragmir-chat/package.json

@@ -64,12 +64,12 @@
     "test:coverage": "vitest run --coverage"
   },
   "dependencies": {
-    "node-llama-cpp": "3.19.0"
+    "node-llama-cpp": "3.20.0"
   },
   "devDependencies": {
     "@types/node": "^24.10.1",
-    "publint": "^0.3.21",
+    "publint": "^0.3.23",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.15"
+    "vitest": "^4.1.10"
   }
 }

packages/ragmir-chat/src/index.ts

@@ -13,6 +13,7 @@ import {
   DEFAULT_CHAT_MODEL_PATH,
   DEFAULT_CHAT_PROFILE,
   inspectChatModel,
+  NODE_LLAMA_RUNTIME_VERSION,
   resolveChatModelPaths,
   setupChatModelFiles,
 } from "./profiles.js"
@@ -52,7 +53,7 @@ export async function doctor(options: DoctorOptions = {}): Promise<DoctorReport>
   return {
     node: process.versions.node,
     provider: "node-llama-cpp",
-    runtimeVersion: "3.19.0",
+    runtimeVersion: NODE_LLAMA_RUNTIME_VERSION,
     profile,
     defaultProfile: DEFAULT_CHAT_PROFILE,
     defaultModel: DEFAULT_CHAT_MODEL,

packages/ragmir-chat/src/profiles.ts

@@ -11,7 +11,7 @@ import type {
   SetupChatModelResult,
 } from "./types.js"
 
-export const NODE_LLAMA_RUNTIME_VERSION = "3.19.0" as const
+export const NODE_LLAMA_RUNTIME_VERSION = "3.20.0" as const
 export const DEFAULT_CHAT_PROFILE: ChatModelProfile = "fast"
 export const DEFAULT_CHAT_MODEL_PATH = ".ragmir/models/chat"
 export const CHAT_MODEL_MANIFEST_FILE = "manifest.json"

packages/ragmir-chat/src/types.ts

@@ -59,7 +59,7 @@ export interface ChatModelProfileDefinition {
 export interface ChatModelManifest {
   schemaVersion: 1
   provider: "node-llama-cpp"
-  runtimeVersion: "3.19.0"
+  runtimeVersion: "3.20.0"
   profile: ChatModelProfile
   modelId: string
   revision: string
@@ -196,7 +196,7 @@ export interface DoctorOptions {
 export interface DoctorReport {
   node: string
   provider: "node-llama-cpp"
-  runtimeVersion: "3.19.0"
+  runtimeVersion: "3.20.0"
   profile: ChatModelProfile
   defaultProfile: ChatModelProfile
   defaultModel: string