From c1bcd51de1ae5b6bb27708577b39cd11b3fa2f5b Mon Sep 17 00:00:00 2001 From: henryle97 Date: Fri, 7 Aug 2026 14:03:50 +0700 Subject: [PATCH 1/5] chore(security): replace confidential sample data with fictional equivalents Problem. Benchmark docs, ADRs, tests, and configs quoted a real issuer by name along with real audited balance-sheet figures taken from a customer document. Configs and doc links also embedded an absolute path from a development machine, which leaked a local directory layout and username and broke every one of those links for anyone else. What changed. The issuer is now Contoso, the standard fictional company. All sixteen real figures were identified by diffing against the source document rather than by eye, and replaced with perturbed values verified not to collide with the original. Two of them also appeared without thousands separators in test fixtures; those are covered too. The audit firm's PCAOB identifier and the incorporation date are likewise replaced. Absolute paths become repo-relative, and markdown links become root-relative so they resolve from any directory depth. Sample document filenames are now neutral. Contoso is deliberately a single word. The entity-audit tests in test_llm_dataset_generator.py assert on how a multi-word entity's first token is handled, so a two-word replacement would have quietly changed what those tests prove. What did NOT change. No behavior. The corpus codenames avatar, pj-doc, and f1-doc are kept: they identify nothing once the company name and figures are gone, and renaming them would churn config filenames, scripts, and collection names for no privacy gain. Co-Authored-By: Claude Opus 5 (1M context) --- .codex | 0 configs/baseline.yml | 2 +- configs/enrichment/all.yml | 2 +- configs/enrichment/auto-keywords.yml | 2 +- configs/enrichment/auto-questions.yml | 2 +- configs/enrichment/table-context.yml | 2 +- configs/enrichment/table-summary.yml | 2 +- configs/parsers/parse-f1-doc-local.yml | 2 +- configs/parsers/parse-f1-doc.yml | 2 +- configs/parsers/parse-pj-doc-local.yml | 2 +- configs/parsers/parse-pj-doc.yml | 2 +- configs/rag-avatar-eval.yml | 2 +- configs/rag-ir-baseline-ragflow.yml | 2 +- configs/rag-ir-baseline.yml | 2 +- configs/rag-ir-hybrid.yml | 2 +- configs/rag-ir-latest.yml | 8 +- configs/rag-ir-mvp1.yml | 2 +- configs/rag-ir-mvp2-rerank.yml | 2 +- configs/rag-ir-mvp2.yml | 2 +- configs/rag-ir-mvp3-all-docs.yml | 4 +- configs/rag-ir-mvp3.yml | 2 +- configs/rag-ir-rerank.yml | 2 +- configs/rag-ir.yml | 2 +- configs/rag-simple-baseline.yml | 2 +- docs/BENCHMARK-chunk.md | 4 +- docs/BENCHMARK.md | 12 +-- ...0002-retrieval-text-augment-not-replace.md | 4 +- ...gational-testset-primary-chunk-evidence.md | 4 +- docs/chunk_strategy/section_table.md | 10 +-- docs/chunker.md | 8 +- docs/command.md | 6 +- docs/enrichment.md | 22 +++--- docs/indexing/table-summary-ablation.md | 6 +- docs/library.md | 4 +- docs/plans.md | 2 +- docs/rag-baseline.md | 6 +- docs/sub-plans/chunking-plan.md | 4 +- .../sub-plans/eval-dataset-generation-plan.md | 2 +- docs/sub-plans/llm-agent-integration.md | 6 +- docs/sub-plans/multi-doc-rag-mvp-plan.md | 2 +- docs/sub-plans/parser-preprocess-plan.md | 2 +- docs/sub-plans/retrieval-quality-fixes.md | 6 +- .../sub-plans/section-table-chunker-review.md | 4 +- docs/sub-plans/xrag-public-api.md | 18 ++--- docs/survey/claude-rag-survey.md | 38 +++++----- docs/survey/datasets.md | 2 +- .../survey/preprocessing-and-normalization.md | 6 +- examples/01_quickstart.py | 2 +- examples/04_multi_tenant.py | 4 +- examples/08_per_call_overrides.py | 2 +- examples/09_parse_docx_api_vs_local.py | 2 +- scripts/ingest/ir_latest.sh | 8 +- tests/unit/test_chunk_prepare.py | 24 +++--- tests/unit/test_embed_text.py | 4 +- tests/unit/test_enrichment.py | 4 +- tests/unit/test_eval_create_dataset.py | 8 +- tests/unit/test_eval_dataset_review.py | 14 ++-- tests/unit/test_eval_run.py | 8 +- tests/unit/test_llm_dataset_generator.py | 76 +++++++++---------- tests/unit/test_parser_preprocess.py | 28 +++---- tests/unit/test_verify_executors.py | 4 +- tools/dataset_generation/llm.py | 32 ++++---- tools/dataset_generation/template.py | 2 +- xrag/core/generation/openai.py | 10 +-- 64 files changed, 232 insertions(+), 232 deletions(-) delete mode 100644 .codex diff --git a/.codex b/.codex deleted file mode 100644 index e69de29..0000000 diff --git a/configs/baseline.yml b/configs/baseline.yml index 52973eb..b361652 100644 --- a/configs/baseline.yml +++ b/configs/baseline.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/all.yml b/configs/enrichment/all.yml index db849b9..fbc9566 100644 --- a/configs/enrichment/all.yml +++ b/configs/enrichment/all.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/auto-keywords.yml b/configs/enrichment/auto-keywords.yml index f3d4a37..7b8186f 100644 --- a/configs/enrichment/auto-keywords.yml +++ b/configs/enrichment/auto-keywords.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/auto-questions.yml b/configs/enrichment/auto-questions.yml index 85f46c8..de68b0e 100644 --- a/configs/enrichment/auto-questions.yml +++ b/configs/enrichment/auto-questions.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/table-context.yml b/configs/enrichment/table-context.yml index f65e99f..774d3d9 100644 --- a/configs/enrichment/table-context.yml +++ b/configs/enrichment/table-context.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/table-summary.yml b/configs/enrichment/table-summary.yml index 68bcb90..3a73dd2 100644 --- a/configs/enrichment/table-summary.yml +++ b/configs/enrichment/table-summary.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/raw/ir_document/Pj_AVATAR_F_page_260205.docx" + - path: "data/raw/ir_document/ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/parsers/parse-f1-doc-local.yml b/configs/parsers/parse-f1-doc-local.yml index a866082..4435f25 100644 --- a/configs/parsers/parse-f1-doc-local.yml +++ b/configs/parsers/parse-f1-doc-local.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # override with --output-dir sources: - - path: "data/raw/ir_document/F1-file.docx" + - path: "data/raw/ir_document/ir-sample-b.docx" doc_id: f1-doc parser: diff --git a/configs/parsers/parse-f1-doc.yml b/configs/parsers/parse-f1-doc.yml index a2b2120..f11f108 100644 --- a/configs/parsers/parse-f1-doc.yml +++ b/configs/parsers/parse-f1-doc.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # defauloverwrite by --output-dir sources: - - path: "data/raw/ir_document/F1-file.docx" + - path: "data/raw/ir_document/ir-sample-b.docx" doc_id: f1-doc parser: diff --git a/configs/parsers/parse-pj-doc-local.yml b/configs/parsers/parse-pj-doc-local.yml index 59b696a..e2ec9ff 100644 --- a/configs/parsers/parse-pj-doc-local.yml +++ b/configs/parsers/parse-pj-doc-local.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # override with --output-dir sources: - - path: "data/raw/ir_document/Pj_AVATAR_F_page_260205.docx" + - path: "data/raw/ir_document/ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/parsers/parse-pj-doc.yml b/configs/parsers/parse-pj-doc.yml index 5545da4..5a121c3 100644 --- a/configs/parsers/parse-pj-doc.yml +++ b/configs/parsers/parse-pj-doc.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # defauloverwrite by --output-dir sources: - - path: "Pj_AVATAR_F_page_260205.docx" + - path: "ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/rag-avatar-eval.yml b/configs/rag-avatar-eval.yml index 1e9770c..0aa2299 100644 --- a/configs/rag-avatar-eval.yml +++ b/configs/rag-avatar-eval.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-baseline-ragflow.yml b/configs/rag-ir-baseline-ragflow.yml index 7378c0d..a75eaf2 100644 --- a/configs/rag-ir-baseline-ragflow.yml +++ b/configs/rag-ir-baseline-ragflow.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-baseline.yml b/configs/rag-ir-baseline.yml index 7b5e19b..e53eb10 100644 --- a/configs/rag-ir-baseline.yml +++ b/configs/rag-ir-baseline.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-hybrid.yml b/configs/rag-ir-hybrid.yml index e5a286e..bcd7d78 100644 --- a/configs/rag-ir-hybrid.yml +++ b/configs/rag-ir-hybrid.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-latest.yml b/configs/rag-ir-latest.yml index e42287c..c620c89 100644 --- a/configs/rag-ir-latest.yml +++ b/configs/rag-ir-latest.yml @@ -1,15 +1,15 @@ pipeline: name: rag-ir-all-docs-latest mode: full - artifact_dir: /data/lucas/projects/rag/ir_v2/data/dev/ir_document + artifact_dir: data/dev/ir_document tenant_id: dev doc_ids: - f1-doc - pj-doc sources: -- path: /data/lucas/projects/rag/ir_v2/data/dev/ir_document/raw/F1-file.docx +- path: data/dev/ir_document/raw/ir-sample-b.docx doc_id: f1-doc -- path: /data/lucas/projects/rag/ir_v2/data/dev/ir_document/raw/Pj_AVATAR_F_page_260205.docx +- path: data/dev/ir_document/raw/ir-sample-a.docx doc_id: pj-doc parser: provider: unstructured_local @@ -53,7 +53,7 @@ vector_store: provider: qdrant options: collection_name: ir_document_latest - path: /data/lucas/projects/rag/ir_v2/data/dev/ir_document/qdrant_db + path: data/dev/ir_document/qdrant_db distance: cosine indexing: exclude_parents_from_index: true diff --git a/configs/rag-ir-mvp1.yml b/configs/rag-ir-mvp1.yml index 4a8b637..b5a2d73 100644 --- a/configs/rag-ir-mvp1.yml +++ b/configs/rag-ir-mvp1.yml @@ -22,7 +22,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-mvp2-rerank.yml b/configs/rag-ir-mvp2-rerank.yml index 6b8d2d5..4851353 100644 --- a/configs/rag-ir-mvp2-rerank.yml +++ b/configs/rag-ir-mvp2-rerank.yml @@ -16,7 +16,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-mvp2.yml b/configs/rag-ir-mvp2.yml index f0d9325..db07247 100644 --- a/configs/rag-ir-mvp2.yml +++ b/configs/rag-ir-mvp2.yml @@ -14,7 +14,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-mvp3-all-docs.yml b/configs/rag-ir-mvp3-all-docs.yml index 637c6b8..23c46a2 100644 --- a/configs/rag-ir-mvp3-all-docs.yml +++ b/configs/rag-ir-mvp3-all-docs.yml @@ -8,9 +8,9 @@ pipeline: doc_ids: [f1-doc, pj-doc] sources: - - path: "data/raw/ir_document/F1-file.docx" + - path: "data/raw/ir_document/ir-sample-b.docx" doc_id: f1-doc - - path: "data/raw/ir_document/Pj_AVATAR_F_page_260205.docx" + - path: "data/raw/ir_document/ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/rag-ir-mvp3.yml b/configs/rag-ir-mvp3.yml index 60cdc13..56681c2 100644 --- a/configs/rag-ir-mvp3.yml +++ b/configs/rag-ir-mvp3.yml @@ -17,7 +17,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-rerank.yml b/configs/rag-ir-rerank.yml index 4eadf4b..bb8395f 100644 --- a/configs/rag-ir-rerank.yml +++ b/configs/rag-ir-rerank.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir.yml b/configs/rag-ir.yml index 6c900c6..0a2034e 100644 --- a/configs/rag-ir.yml +++ b/configs/rag-ir.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-simple-baseline.yml b/configs/rag-simple-baseline.yml index b20f10d..4512371 100644 --- a/configs/rag-simple-baseline.yml +++ b/configs/rag-simple-baseline.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/docs/BENCHMARK-chunk.md b/docs/BENCHMARK-chunk.md index f842e63..d36e389 100644 --- a/docs/BENCHMARK-chunk.md +++ b/docs/BENCHMARK-chunk.md @@ -113,7 +113,7 @@ All on gpt-4.1-nano for enrichment + generation, `text-embedding-3-small` for em 1. **`table_summary` did NOT help on this testset.** Clean A/B at the same retrieval stack (rows 04b vs 05) shows `num_match` *regressed* by −0.037 with the summary on, MRR was effectively flat (−0.003), substring tied. Earlier "row 04 vs row 01" delta was *real* but attributable to the hybrid retrieval stack (BM25 + RRF + dedup_family), not the summary. 2. **Cohere reranker did NOT help on this testset.** Rows 04 vs 04b show `num_match` flat (0.926 = 0.926), MRR +0.002, substring *worse* −0.026 with rerank on. Rerank costs +3651ms p95 retrieval latency for negative quality movement on a 39-q testset. 3. **The hybrid retrieval stack alone (BM25 + vector + RRF + dedup_family)** is the lever responsible for the gains over the simple-retrieval baseline. Row 05 (no summary, no rerank, just hybrid) lifts num_match from 0.852 to 0.963 vs row 01 — a +0.111 gain attributable purely to the retriever change. Substring also +0.077, MRR +0.051. With ~no latency cost vs simple (450ms vs 437ms p95). -4. **Likely cause for the summary regression**: hallucinated numbers in summaries (observed in the inspection — chunk:56 "hundreds of millions" vs actual $8K–$438K; chunk:129 "$672M" vs actual ~$672K) shift the embedding into the wrong neighborhood. The LLM at prompt time still reads correct HTML, but it gets the *wrong table* fed in because retrieval ranked elsewhere. `auto_questions` (HyQE) is doing better natural-language anchoring without the numeric hallucination risk because the questions are query-shaped and don't have to commit to specific magnitudes. +4. **Likely cause for the summary regression**: hallucinated numbers in summaries (observed in the inspection — chunk:56 "hundreds of millions" vs actual $8K–$471K; chunk:129 "$540M" vs actual ~$540K) shift the embedding into the wrong neighborhood. The LLM at prompt time still reads correct HTML, but it gets the *wrong table* fed in because retrieval ranked elsewhere. `auto_questions` (HyQE) is doing better natural-language anchoring without the numeric hallucination risk because the questions are query-shaped and don't have to commit to specific magnitudes. 5. **`top_k_redundancy` improvements** (0.431 → 0.285–0.295 across rows 04/04b/05) are entirely from `dedup_family: true`, not from any enrichment. ### Practical takeaways @@ -161,7 +161,7 @@ All on gpt-4.1-nano for enrichment + generation, `text-embedding-3-small` for em string fragment across multiple chunks. 6. **`exact_match=0.000` everywhere.** The generator's normalized answer never matches the gold string verbatim. Expected — `gpt-4.1-nano` - paraphrases ("$7,351,900" vs "$7.35M"). Use `numeric_match` / + paraphrases ("$6,120,400" vs "$6.12M"). Use `numeric_match` / `substring_match` as the answer-grounding signal. ## Caveats / methodology notes diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index 1bb5c16..ba04157 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -49,7 +49,7 @@ testset: from it. **A correct refusal IS the right answer**, so a low score here is a metric artifact, not a generation defect. 2. **Terse factual answers**: byte-exact replies like `"17%"`, - `"January 2025"`, `"$4,431,280"` can be the answer to many unrelated + `"January 2025"`, `"$3,845,200"` can be the answer to many unrelated questions, so the synthetic-question step finds many low-similarity candidates and drives the mean down. Several **exact-match** answers in the multi-doc 87-Q set (1658 run) score 0.13–0.20. @@ -165,7 +165,7 @@ count would bring it under 1000ms. --- -## IR document (AVATAR / Neksai) — 40 LLM-generated questions +## IR document (AVATAR / Contoso) — 40 LLM-generated questions Dataset: `data/artifacts/ir_document/eval-llm/testset.jsonl` (40 Qs, 39 valid after validation). Categories: table_lookup (8), direct_lookup (15), semantic_paraphrase (8), multi_hop (6), no_answer (3). @@ -221,7 +221,7 @@ Embedding: `text-embedding-3-small` · Judge: `gpt-4.1-mini` · RAGAS metrics: ` | 01 | rag-ir-mvp1 | section_table | hybrid | 10 | gpt-4.1-mini | 0.973 | **0.96** | **0.83** | — | 543ms | **6267ms** | 622s | [100q-20260424-1126](../experiments/eval/rag-ir-mvp1-100q-20260424-1126/) | | 02 | rag-ir-mvp2 | section_table | hybrid | 10 | gpt-4.1-mini | 0.973 | **0.96** | 0.82 | — | **501ms** | 62791ms§ | 641s | [100q-20260424-1137](../experiments/eval/rag-ir-mvp2-100q-20260424-1137/) | -§ MVP2 `e2e p95=62791ms` is inflated by 5 queries with 60–91s generation times. Spot-check shows correct short answers (e.g. "$22,265,412" in 105 chars) — these are transient `gpt-4.1-mini` API stalls, not prompt-size or retrieval issues. MVP1 (same generator, ran 10 min earlier) saw max gen=9746ms. **Re-run recommended** to confirm MVP2 is not systematically slower. +§ MVP2 `e2e p95=62791ms` is inflated by 5 queries with 60–91s generation times. Spot-check shows correct short answers (e.g. "$24,830,170" in 105 chars) — these are transient `gpt-4.1-mini` API stalls, not prompt-size or retrieval issues. MVP1 (same generator, ran 10 min earlier) saw max gen=9746ms. **Re-run recommended** to confirm MVP2 is not systematically slower. ### MVP1 vs MVP2 on 100 Qs @@ -241,7 +241,7 @@ Quality is a statistical tie. The RAGFlow-style indexing changes in MVP2 (parent ## PJ-doc — 48 LLM-generated questions -New corpus (PJ doc, distinct from AVATAR/Neksai). Indexed into Qdrant collection +New corpus (PJ doc, distinct from AVATAR/Contoso). Indexed into Qdrant collection `ir_dev_mvp3_table_summary_v1` from `data/dev/ir_document/pj-doc/chunked-section-table/chunks.json` (174 chunks: 60 section, 24 table, 90 text). @@ -390,7 +390,7 @@ near-perfect on recall. | **Faithfulness jumps 0.92 → 0.96 on 100 Qs vs 40 Qs** | Larger testset has a more balanced mix; multi_hop (the 40Q drag, fa=0.72) is proportionally less represented. Not a regression of the smaller testset. | | **`gpt-4.1-mini` has sporadic multi-second gen stalls** | MVP2-100q had 5 queries at 60–91s gen (vs MVP1-100q max 9.7s same day). Answers correct and short — provider-side tail, not prompt-size. Inflates e2e p95; mean/median unaffected. | | **`gpt-4.1-nano` matches `gpt-4.1-mini` quality at ~1/3 cost** (40 Qs) | Row 07 (mini, MVP2) fa=0.93 ar=0.78 cp=0.81 vs Row 08 (nano, MVP2) fa=0.93 ar=0.81 cp=0.77 — within noise on all RAGAS metrics. Nano tightens e2e p95 from 5556ms → 2243ms (new best in the table). Re-run at 100 Qs recommended before promoting to MVP 1. | -| **PJ-doc is a harder corpus than AVATAR/Neksai on the same stack** | Same `rag-ir-mvp3-table-summary` config: AVATAR 40-Q (proxy: 39-Q rerank-on row, fa≈0.95 historical) vs PJ-doc 48-Q (row 02: fa=0.884, ar=0.706, cp=0.779). ans_rel drops ~0.10 — PJ-doc questions are tougher to answer concisely, even with element-level retrieval at 100%. Multi-hop (7 Qs) and the wider `needs_review` validation rate (16/48) are the likely driver. | +| **PJ-doc is a harder corpus than AVATAR/Contoso on the same stack** | Same `rag-ir-mvp3-table-summary` config: AVATAR 40-Q (proxy: 39-Q rerank-on row, fa≈0.95 historical) vs PJ-doc 48-Q (row 02: fa=0.884, ar=0.706, cp=0.779). ans_rel drops ~0.10 — PJ-doc questions are tougher to answer concisely, even with element-level retrieval at 100%. Multi-hop (7 Qs) and the wider `needs_review` validation rate (16/48) are the likely driver. | | **`element_*` metrics are the right signal on LLM-gen testsets with overlapping chunks** | PJ-doc row 02: `retrieval_recall=0.761` but `element_recall=1.000`, `element_hit=1.000`, `element_mrr@10=0.893`. The LLM generator labels one chunk-id as evidence, but the retriever often returns a sibling chunk containing the same elements. Treat `element_recall`/`element_mrr` as the truth for retrieval health on this kind of testset. | | **Reranker is correctly off by default for inner-loop iteration** (PJ-doc) | Empirically validated by the 39-Q comparison cited in `configs/rag-ir-mvp3-table-summary.yml`: Cohere `rerank-v3.5` adds 6.6× retrieval p95 (649ms → 4300ms) for ~0 quality gain (`element_mrr@10` 0.887 vs 0.889, `substring_match` regressed −0.026). Flip on only for promotion-grade runs. | | **Multi-doc element_recall holds at 0.967 (no per-question doc filter)** | Multi-doc row 02 (PJ 50 + F1 49 = 99 Qs) on the combined corpus, both docs visible to the retriever: `element_recall=0.967`, `element_mrr@10=0.724`. Per-doc: PJ 0.880 / F1 0.898. PJ has 46% cross-doc top-10 leakage (smaller corpus, F1 chunks crowd in) and F1 has 14% — both still find chunks carrying the right source elements ~9/10 times. Cross-doc retrieval is *intentional* here, not a fix-it. The earlier 0.494 element_recall (row 01) was a stale-testset artifact: PJ's `evidence_element_ids` were generated against the API parse and didn't survive the local re-parse (0/77 element_ids matched). Regenerating the PJ testset against the local-parsed chunks fixed the metric without any retrieval changes. **Lesson: re-run testset generation any time the parser changes; chunk_ids stay stable but element_ids do not.** | @@ -400,7 +400,7 @@ near-perfect on recall. | **Indexing-time chunk hygiene removes index pollution** (758 vs 778 prior) | New `IndexingConfig` (`min_section_body_chars=30`, `merge_continuation_sections=True`) drops heading-only sections and merges page-split `(cont.)` siblings before parent exclusion: pj-doc 173 → 165 (8 cont. merged) → 113 indexed; f1-doc 840 → 645 indexed (no cont. sections). Cleaner top-k (no embedding budget on `"10. TAXATION"` or cover-page metadata), and `(cont.)` family dedup now treats split sections as one unit. | | **`exclude_parents_from_index` moved from env var to YAML** (`AppConfig.indexing`) | `Settings.exclude_parents_from_index` (env var `EXCLUDE_PARENTS_FROM_INDEX`) removed in favor of YAML `indexing.exclude_parents_from_index` (default `True`). Configs are now the single source of truth for "how chunks were indexed"; no per-process env to forget. Two new sibling fields land in the same block: `min_section_body_chars` and `merge_continuation_sections`. | | **Enrichment concurrency (max_concurrency=10) is benchmark-neutral** | Multi-doc row 05 (20260504-1511, conc=10) vs row 13 compile log (20260427-0523, conc=1, same 86-Q run): Recall@10 0.937 vs 0.924 (+0.013), MRR@10 0.621 vs 0.641 (−0.020), num_match 0.923 vs 0.897 (+0.026). All deltas within LLM generation variance at temperature=0.1. PR #39 (per-step `batch_as_completed`) correctness confirmed. | -| **ADR 0002 augment-not-replace is benchmark-neutral on entity-prefixed testsets** (Multi-doc 95 Qs) | Multi-doc row 06 (20260505-1054) vs prior 95-Q best (20260504-1603): R@10 0.994 → 0.983 (−0.011), element_recall 1.000 → 0.989 (−0.011), MRR@10 0.723 → 0.724 (+0.001), num_match 0.750 → 0.750 (flat), retrieval p95 618ms → 433ms (−30%). All deltas within loose tolerance. The one flipped question (`qa:0011`: *"What was the cash balance as of December 31, 2023…"*) is a cell-value table query — exactly one of the two failure modes ADR 0002 targets — and regressed from rank 9 → out-of-top-10 because augmenting bodies into all chunks made f1-doc text chunks more competitive on this query, out-ranking the pj-doc table chunk despite its `Cash \| $4,431,280` now being in the dense vector. **Lesson: this LLM-gen testset (auto-question-derived → entity-prefixed) cannot witness the navigational/cell-value wins ADR 0002 targets; the deferred integration regression pins (PR #45 follow-up) are the right gate.** See `experiments/eval/rag-ir-mvp3-all-docs-20260505-1054/fail_cases.md`. | +| **ADR 0002 augment-not-replace is benchmark-neutral on entity-prefixed testsets** (Multi-doc 95 Qs) | Multi-doc row 06 (20260505-1054) vs prior 95-Q best (20260504-1603): R@10 0.994 → 0.983 (−0.011), element_recall 1.000 → 0.989 (−0.011), MRR@10 0.723 → 0.724 (+0.001), num_match 0.750 → 0.750 (flat), retrieval p95 618ms → 433ms (−30%). All deltas within loose tolerance. The one flipped question (`qa:0011`: *"What was the cash balance as of December 31, 2023…"*) is a cell-value table query — exactly one of the two failure modes ADR 0002 targets — and regressed from rank 9 → out-of-top-10 because augmenting bodies into all chunks made f1-doc text chunks more competitive on this query, out-ranking the pj-doc table chunk despite its `Cash \| $3,845,200` now being in the dense vector. **Lesson: this LLM-gen testset (auto-question-derived → entity-prefixed) cannot witness the navigational/cell-value wins ADR 0002 targets; the deferred integration regression pins (PR #45 follow-up) are the right gate.** See `experiments/eval/rag-ir-mvp3-all-docs-20260505-1054/fail_cases.md`. | | **Latest 124-Q row proves the navigational slice is the missing witness** | Multi-doc row 07 (`2026-05-05 14:54 UTC`, `data/dev/ir_document/latest`): legacy categories stay high on recall — direct_lookup `1.000`, semantic_paraphrase `1.000`, table_lookup `0.958`, multi_hop `0.806` — while `navigational` drops to `0.600` R@10 and `0.221` MRR@10. The L2 gate passes cleanly: `0.600 < median(1.000, 1.000, 0.958, 0.806) = 0.979`. Concrete miss cluster: all three `Our Strategy` navigational rows miss their pinned chunk entirely, with `SPECIAL NOTE REGARDING FORWARD-LOOKING STATEMENTS` and TOC-like sections crowding the top ranks instead. See `experiments/eval/rag-ir-mvp3-all-docs-20260505-1454/fail_cases.md`. | --- diff --git a/docs/adr/0002-retrieval-text-augment-not-replace.md b/docs/adr/0002-retrieval-text-augment-not-replace.md index 0c2d2b6..5c7d7d8 100644 --- a/docs/adr/0002-retrieval-text-augment-not-replace.md +++ b/docs/adr/0002-retrieval-text-augment-not-replace.md @@ -31,10 +31,10 @@ both query and document. Our corpus is F-1 / project-document filings. Users ask **navigational** questions like *"how is the corporate structured?"* without naming the issuer. Auto-generated questions all start with the entity name (*"What -strategic corporate restructuring steps did Neksai undertake…"*), so +strategic corporate restructuring steps did Contoso undertake…"*), so entity-less queries miss the dense vector entirely. Issue #43 documents the failure case: `f1-doc:chunk:84` (the actual structural narrative -naming Neksai Inc., Neksai BVI, Neksai SG) never appears in top-10 +naming Contoso Inc., Contoso BVI, Contoso SG) never appears in top-10 for the query *"how is the corporate structured"*, but does for the entity-injected variant *"how is nexksai corporate structured"*. diff --git a/docs/adr/0003-navigational-testset-primary-chunk-evidence.md b/docs/adr/0003-navigational-testset-primary-chunk-evidence.md index 3031126..2a35899 100644 --- a/docs/adr/0003-navigational-testset-primary-chunk-evidence.md +++ b/docs/adr/0003-navigational-testset-primary-chunk-evidence.md @@ -19,8 +19,8 @@ shares: **multiple chunks legitimately satisfy the question**. For *"how is the corporate structured?"* against the f1-doc corpus, the following chunks are all valid retrieval results: -- `f1-doc:chunk:84` — leaf naming the subsidiaries (*Neksai Inc.*, - *Neksai BVI*, *Neksai SG*) +- `f1-doc:chunk:84` — leaf naming the subsidiaries (*Contoso Inc.*, + *Contoso BVI*, *Contoso SG*) - the parent section under *"CORPORATE HISTORY AND STRUCTURE"* — describes the structure narratively - a hypothetical diagram chunk (currently dropped by the parser) — diff --git a/docs/chunk_strategy/section_table.md b/docs/chunk_strategy/section_table.md index 2b39a4e..d36fcb1 100644 --- a/docs/chunk_strategy/section_table.md +++ b/docs/chunk_strategy/section_table.md @@ -339,8 +339,8 @@ paragraphs>"` truncated to `section_content_max_chars`. { "chunk_type": "table", "table_html": "...", - "table_text": "Cash | $4,431,280 | $7,351,900", - "table_text_markdown": "| Cash | $4,431,280 | $7,351,900 |\n| --- | --- | --- |", + "table_text": "Cash | $3,845,200 | $6,120,400", + "table_text_markdown": "| Cash | $3,845,200 | $6,120,400 |\n| --- | --- | --- |", "pre_text": "...", // joined neighbor text (above) "post_text": "...", // joined neighbor text (below) "token_count": @@ -444,9 +444,9 @@ Input elements (post-preprocess): [ {"type": "Title", "text": "1. ORGANIZATION", "element_id": "h1", "metadata": {"category_depth": 0, "page_number": 7}}, {"type": "Title", "text": "(a) Nature of operations","element_id": "h2", "metadata": {"category_depth": 1, "page_number": 7}}, - {"type": "NarrativeText","text": "Neksai Inc. (the \"Company\") was incorporated in Delaware on January 4, 2020.","element_id": "p1", "metadata": {"page_number": 7}}, + {"type": "NarrativeText","text": "Contoso Inc. (the \"Company\") was incorporated in Delaware on March 12, 2019.","element_id": "p1", "metadata": {"page_number": 7}}, {"type": "NarrativeText","text": "(Stated in U.S. Dollars (\"US$\"))","element_id": "p2", "metadata": {"page_number": 8}}, - {"type": "Table", "text": "Cash | $4,431,280 | $7,351,900","element_id": "t1", "metadata": {"page_number": 8, "text_as_html": "
...
"}} + {"type": "Table", "text": "Cash | $3,845,200 | $6,120,400","element_id": "t1", "metadata": {"page_number": 8, "text_as_html": "...
"}} ] ``` @@ -454,7 +454,7 @@ Trace: 1. `Title "1. ORGANIZATION"` (depth 0) → flush (no-op), heading_path = `["1. ORGANIZATION"]`, register parent `:chunk:1`. 2. `Title "(a) Nature of operations"` (depth 1) → flush (no-op), heading_path = `["1. ORGANIZATION", "(a) Nature of operations"]`, register parent `:chunk:2`. -3. `NarrativeText "Neksai Inc. ..."` → append to `active_text_group`. +3. `NarrativeText "Contoso Inc. ..."` → append to `active_text_group`. 4. `NarrativeText "(Stated in U.S. Dollars ("US$"))"` → matches parenthetical-caption-shaped → still treated as text → append to `active_text_group`. *(Note: this only matters when the element type is `Title`/`Header`, the regex check is gated upstream. As a `NarrativeText`, it just goes to the group naturally.)* 5. `Table` → `flush_text_group()` emits `:chunk:3` (text child of parent 2) with the two prose paragraphs joined by `\n`. Then `_build_table_chunk` collects `p2` as `pre_elements` (1 element, less than max=2; stops at Title boundary backward). No `post_elements` (end of input). Emits `:chunk:4` (table child of parent 2). 6. End of input → final `flush_text_group()` no-op. diff --git a/docs/chunker.md b/docs/chunker.md index 11f46e7..d76e68d 100644 --- a/docs/chunker.md +++ b/docs/chunker.md @@ -241,7 +241,7 @@ Table chunks add: `table_html`, `table_text`, `table_text_markdown`, `pre_text`, "chunk_type": "text", "chunk_strategy": "title_hierarchy", "heading_path": ["1. ORGANIZATION", "(a) Nature of operations"], - "content": "1. ORGANIZATION\n(a) Nature of operations\nNeksai Inc. (the \"Company\") was incorporated ...", + "content": "1. ORGANIZATION\n(a) Nature of operations\nContoso Inc. (the \"Company\") was incorporated ...", "source_element_ids": ["t1", "t2", "n1"], "page_number": null, "parent_chunk_id": null @@ -257,13 +257,13 @@ Table chunks add: `table_html`, `table_text`, `table_text_markdown`, `pre_text`, "chunk_type": "table", "chunk_strategy": "section_token", "heading_path": ["CONSOLIDATED BALANCE SHEETS"], - "content": "Section: CONSOLIDATED BALANCE SHEETS\n\nPre-context:\n...\n\nTable:\nCash | $4,431,280 | $7,351,900\n\nPost-context:\n...", + "content": "Section: CONSOLIDATED BALANCE SHEETS\n\nPre-context:\n...\n\nTable:\nCash | $3,845,200 | $6,120,400\n\nPost-context:\n...", "source_element_ids": ["table-element-1"], "page_number": 8, "parent_chunk_id": "avatar_comments_doc:chunk:11", "table_html": "...
", - "table_text": "Cash | $4,431,280 | $7,351,900", - "table_text_markdown": "| Cash | $4,431,280 | $7,351,900 |\n| --- | --- | --- |", + "table_text": "Cash | $3,845,200 | $6,120,400", + "table_text_markdown": "| Cash | $3,845,200 | $6,120,400 |\n| --- | --- | --- |", "pre_text": "", "post_text": "Giving retroactive effect to the issuance of 1 ordinary share..." } diff --git a/docs/command.md b/docs/command.md index b692a54..282ac17 100644 --- a/docs/command.md +++ b/docs/command.md @@ -48,7 +48,7 @@ Example: uv run python -m src.cli convert \ --source docx \ --target pdf \ - --input "data/Pj AVATAR_F page (260205)-with comments.docx" \ + --input "data/ir-sample-a.docx" \ --output data/raw/avatar-comments.pdf ``` @@ -198,7 +198,7 @@ Typical use: Build structured retrieval chunks from normalized parser elements. Detailed chunker behavior and output schema are documented in -[`docs/chunker.md`](/data/lucas/projects/rag/ir_v2/docs/chunker.md). +[`docs/chunker.md`](/docs/chunker.md). Help: @@ -441,7 +441,7 @@ Behavior: Run one-shot baseline RAG over a local `chunks.json` file. Detailed behavior is documented in -[`docs/rag-baseline.md`](/data/lucas/projects/rag/ir_v2/docs/rag-baseline.md). +[`docs/rag-baseline.md`](/docs/rag-baseline.md). Help: diff --git a/docs/enrichment.md b/docs/enrichment.md index 975a12e..e8d0c58 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -60,12 +60,12 @@ enrichment: If `pipeline` is empty, indexing behaves as before and embeds plain chunk `content`. -Ready-made configs live under [`configs/enrichment/`](/data/lucas/projects/rag/ir_v2/configs/enrichment): +Ready-made configs live under [`configs/enrichment/`](/configs/enrichment): -- [`table-context.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/table-context.yml) -- [`auto-keywords.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/auto-keywords.yml) -- [`auto-questions.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/auto-questions.yml) -- [`all.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/all.yml) +- [`table-context.yml`](/configs/enrichment/table-context.yml) +- [`auto-keywords.yml`](/configs/enrichment/auto-keywords.yml) +- [`auto-questions.yml`](/configs/enrichment/auto-questions.yml) +- [`all.yml`](/configs/enrichment/all.yml) ## Preview Output @@ -195,9 +195,9 @@ It does not embed text or call the vector store. ## Related Files -- [`xrag/core/enrichment/`](/data/lucas/projects/rag/ir_v2/xrag/core/enrichment) -- [`scripts/enrichment/preview.py`](/data/lucas/projects/rag/ir_v2/scripts/enrichment/preview.py) -- [`scripts/enrichment/show_chunks.py`](/data/lucas/projects/rag/ir_v2/scripts/enrichment/show_chunks.py) -- [`xrag/pipelines/indexing.py`](/data/lucas/projects/rag/ir_v2/xrag/pipelines/indexing.py) -- [`docs/command.md`](/data/lucas/projects/rag/ir_v2/docs/command.md) -- [`docs/sub-plans/ragflow-port-plan.md`](/data/lucas/projects/rag/ir_v2/docs/sub-plans/ragflow-port-plan.md) +- [`xrag/core/enrichment/`](/xrag/core/enrichment) +- [`scripts/enrichment/preview.py`](/scripts/enrichment/preview.py) +- [`scripts/enrichment/show_chunks.py`](/scripts/enrichment/show_chunks.py) +- [`xrag/pipelines/indexing.py`](/xrag/pipelines/indexing.py) +- [`docs/command.md`](/docs/command.md) +- [`docs/sub-plans/ragflow-port-plan.md`](/docs/sub-plans/ragflow-port-plan.md) diff --git a/docs/indexing/table-summary-ablation.md b/docs/indexing/table-summary-ablation.md index 122e25b..fabbf0f 100644 --- a/docs/indexing/table-summary-ablation.md +++ b/docs/indexing/table-summary-ablation.md @@ -120,9 +120,9 @@ Two things conspire: currently-noisy `table_html` (10+ pad columns, split currency cells, repeated multi-row headers). Inspection caught: - `chunk:56`: "hundreds of millions of units" vs. actual $8,810 - (2023) / $438,533 (2024). - - `chunk:129`: "total lease payments of $672.2 million" vs. actual - ~$672K — three orders of magnitude off. + (2023) / $471,280 (2024). + - `chunk:129`: "total lease payments of $540.3 million" vs. actual + ~$540K — three orders of magnitude off. - `chunk:170`: "yen" labels on USD figures. - `chunk:21`: "$0.01 par value" vs. actual $0.00001 (post-subdivision). 2. **Option B drops raw cells from the embed text for tables** — the diff --git a/docs/library.md b/docs/library.md index b3e8e93..d53bea4 100644 --- a/docs/library.md +++ b/docs/library.md @@ -38,7 +38,7 @@ client = Xrag( generation="openai:gpt-4.1-nano", ) -doc = await client.documents.ingest("F1-file.docx", collection="ir_docs") +doc = await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") result = await client.rag.ask( query="What is this document about?", collection="ir_docs", @@ -58,7 +58,7 @@ Runnable example: ```python doc = await client.documents.ingest( - "F1-file.docx", + "ir-sample-b.docx", collection="ir_docs", ) ``` diff --git a/docs/plans.md b/docs/plans.md index 7609143..44451c8 100644 --- a/docs/plans.md +++ b/docs/plans.md @@ -252,7 +252,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/docs/rag-baseline.md b/docs/rag-baseline.md index 30ea8f7..ac89786 100644 --- a/docs/rag-baseline.md +++ b/docs/rag-baseline.md @@ -76,7 +76,7 @@ generation: The included config file is: -- [`configs/rag-simple-baseline.yml`](/data/lucas/projects/rag/ir_v2/configs/rag-simple-baseline.yml) +- [`configs/rag-simple-baseline.yml`](/configs/rag-simple-baseline.yml) ## Input @@ -110,7 +110,7 @@ The command prints JSON: { "input_path": "data/artifacts//chunked/chunks.json", "query": "What was Cash as of December 31, 2024?", - "answer": "$7,351,900", + "answer": "$6,120,400", "model": "gpt-4.1-mini", "indexed_chunks": 129, "retrieved_chunks": 5, @@ -122,7 +122,7 @@ The command prints JSON: Code lives in: -- [`xrag/pipelines/rag_baseline.py`](/data/lucas/projects/rag/ir_v2/xrag/pipelines/rag_baseline.py) +- [`xrag/pipelines/rag_baseline.py`](/xrag/pipelines/rag_baseline.py) It uses: diff --git a/docs/sub-plans/chunking-plan.md b/docs/sub-plans/chunking-plan.md index 38f15a5..709824d 100644 --- a/docs/sub-plans/chunking-plan.md +++ b/docs/sub-plans/chunking-plan.md @@ -95,8 +95,8 @@ Practical takeaway: Local references: -- [`docs/dataset-formats.md`](/data/lucas/projects/rag/ir_v2/docs/dataset-formats.md) -- [`docs/survey/datasets.md`](/data/lucas/projects/rag/ir_v2/docs/survey/datasets.md) +- [`docs/dataset-formats.md`](/docs/dataset-formats.md) +- [`docs/survey/datasets.md`](/docs/survey/datasets.md) ## Recommended Chunk Model diff --git a/docs/sub-plans/eval-dataset-generation-plan.md b/docs/sub-plans/eval-dataset-generation-plan.md index 1867806..97d17e4 100644 --- a/docs/sub-plans/eval-dataset-generation-plan.md +++ b/docs/sub-plans/eval-dataset-generation-plan.md @@ -580,7 +580,7 @@ Each final JSONL row should follow a stable schema like: "doc_id": "avatar_comments_doc", "chunk_strategy": "section_table", "question": "What was Cash as of December 31, 2024?", - "answer": "$7,351,900", + "answer": "$6,120,400", "answer_type": "numeric", "reasoning_type": "single_hop_table", "difficulty": "easy", diff --git a/docs/sub-plans/llm-agent-integration.md b/docs/sub-plans/llm-agent-integration.md index 2aa5f34..17c4042 100644 --- a/docs/sub-plans/llm-agent-integration.md +++ b/docs/sub-plans/llm-agent-integration.md @@ -3,7 +3,7 @@ **Status:** draft (rewrite of 2026-04-26 Path-A plan; awaiting Phase-0 kickoff) **Author:** Claude (planning pass, 2026-04-27) **Companion docs:** [`api/apis_design.md`](api/apis_design.md), [`api/phase1.md`](api/phase1.md), [`api/phase1_review.md`](api/phase1_review.md), [`api/system.md`](api/system.md), [`ingest-module-plan.md`](ingest-module-plan.md), [`../../llm-agent/docs/langchain-compat-audit.md`](../../../llm-agent/docs/langchain-compat-audit.md) -**Target consumer repo:** `/data/lucas/projects/rag/llm-agent` (FastAPI + Django ORM + RabbitMQ; Python 3.11; uv) +**Target consumer repo:** `/path/to/llm-agent` (FastAPI + Django ORM + RabbitMQ; Python 3.11; uv) **Engine repo:** `henryle97/xrag` (this repo, ir_v2/; renamed from `henryle97/fin-rag` on 2026-04-27) --- @@ -225,7 +225,7 @@ This test is the gate for tagging v0.1.0. uv build # → dist/xrag-0.1.0-py3-none-any.whl uv venv --python 3.12 /tmp/xrag-test /tmp/xrag-test/bin/uv pip install \ - "/data/lucas/projects/rag/ir_v2/dist/xrag-0.1.0-py3-none-any.whl[rerank-cohere]" + "dist/xrag-0.1.0-py3-none-any.whl[rerank-cohere]" /tmp/xrag-test/bin/python -c "from xrag import run_rag_query, run_ingest_pipeline; print('ok')" ``` @@ -372,7 +372,7 @@ Self-contained, does not piggyback on `EventManagerConfig`. Reproduce the dev-run shape via the new API end-to-end: -1. `POST /api/v1/rag/documents` with `data/raw/ir_document/F1-file.docx` → status reaches `ready` within 2 minutes. +1. `POST /api/v1/rag/documents` with `data/raw/ir_document/ir-sample-b.docx` → status reaches `ready` within 2 minutes. 2. `POST /api/v1/rag/retrievals` with the 87-question testset queries → assert top-line metrics match `experiments/eval/rag-ir-mvp3-all-docs-20260426-1658/summary.txt` within noise: - `deterministic.retrieval_hit ≥ 0.99` - `deterministic.recall_at_10 ≥ 0.97` diff --git a/docs/sub-plans/multi-doc-rag-mvp-plan.md b/docs/sub-plans/multi-doc-rag-mvp-plan.md index 52563c0..e7eddcc 100644 --- a/docs/sub-plans/multi-doc-rag-mvp-plan.md +++ b/docs/sub-plans/multi-doc-rag-mvp-plan.md @@ -335,7 +335,7 @@ Add 5–10 questions the auto-gen pipeline can't produce: full-eval mode. - **`num_match` quality.** It's a substring check, not an LLM judge. It will under-count cases where the model paraphrases - the right number ("$4.4M" vs "4,431,280"). Accept this for + the right number ("$3.8M" vs "3,845,200"). Accept this for the inner loop; full eval at promotion catches it. ### Notes diff --git a/docs/sub-plans/parser-preprocess-plan.md b/docs/sub-plans/parser-preprocess-plan.md index 9fd2e31..3ded850 100644 --- a/docs/sub-plans/parser-preprocess-plan.md +++ b/docs/sub-plans/parser-preprocess-plan.md @@ -88,7 +88,7 @@ Recommended heading heuristic: - repeated 3 or more times - short text, e.g. <= 80 chars - not a substantive sentence - - common company/page-header style content such as `NEKSAI INC.` + - common company/page-header style content such as `CONTOSO INC.` 6. `prefer_table_html(elements)` diff --git a/docs/sub-plans/retrieval-quality-fixes.md b/docs/sub-plans/retrieval-quality-fixes.md index 5fe0136..33f7892 100644 --- a/docs/sub-plans/retrieval-quality-fixes.md +++ b/docs/sub-plans/retrieval-quality-fixes.md @@ -14,8 +14,8 @@ defaults: `gpt-4.1-nano` generator, 3 RAGAS metrics, `gpt-4o-mini` judge). | Trace | Query | Expected | Answer | Primary source rank | |---|---|---|---|---| -| 1 | "total rent expenditure for FY 2023" | US$298,754 | ✅ correct | **1** (lucky) | -| 2 | "total cash by December 31, 2024" | US$7,351,900 | ✅ correct | **8** (balance sheet chunk 12) | +| 1 | "total rent expenditure for FY 2023" | US$341,620 | ✅ correct | **1** (lucky) | +| 2 | "total cash by December 31, 2024" | US$6,120,400 | ✅ correct | **8** (balance sheet chunk 12) | Answers are correct, but the ranking reveals three problems that will bite harder as we drop to smaller top-k, introduce reranking, or move @@ -67,7 +67,7 @@ Two compounding causes: has ~40 rows covering 4 line-item categories. BM25's length normalisation penalises long docs; the dense embedding captures a diffuse "financial statement" topic vector rather than a specific - "cash" signal. The answer row ("Cash | $4,431,280 | $7,351,900") is + "cash" signal. The answer row ("Cash | $3,845,200 | $6,120,400") is one of dozens. Today the generator gets the right answer because a *different* chunk diff --git a/docs/sub-plans/section-table-chunker-review.md b/docs/sub-plans/section-table-chunker-review.md index 66193f8..9a92f50 100644 --- a/docs/sub-plans/section-table-chunker-review.md +++ b/docs/sub-plans/section-table-chunker-review.md @@ -312,8 +312,8 @@ Mapping items in | Bucket | Item | Chunker contribution | |---|---|---| | C — wrong-cell (×14) | `ETH 2023 = 290` | S3: gold in chunk:120 (cont.), retrieval saturated by chunks 116-118 under base parent | -| C — wrong-cell | `salary = $(361,993)` | S3 + S2: same fragmentation; nearby paragraph emitted 3× crowds top-k | -| D — retrieval miss (×3) | `digital assets 2023 = $5,868,425` (gold chunk:12 in balance sheet) | S3: section 114+120 carry strong "digital assets" lexical match and out-rank the balance-sheet line | +| C — wrong-cell | `salary = $(405,760)` | S3 + S2: same fragmentation; nearby paragraph emitted 3× crowds top-k | +| D — retrieval miss (×3) | `digital assets 2023 = $6,214,730` (gold chunk:12 in balance sheet) | S3: section 114+120 carry strong "digital assets" lexical match and out-rank the balance-sheet line | | E — found at low rank (×16) | `cash 2024` etc. | S1 + S6: parent/sibling near-duplicates eat top-3 slots, evidence pushed to rank 3-7 | S3 (`(cont.)` fragmentation) is the single highest-leverage finding — diff --git a/docs/sub-plans/xrag-public-api.md b/docs/sub-plans/xrag-public-api.md index 67bd65e..828a2e8 100644 --- a/docs/sub-plans/xrag-public-api.md +++ b/docs/sub-plans/xrag-public-api.md @@ -106,7 +106,7 @@ Concrete patterns observed in current docs (April 2026), via Context7. Survey is from xrag import Xrag client = Xrag.from_env() -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") result = await client.rag.ask("What's the FY-23 revenue?", collection="ir_docs") print(result.answer) ``` @@ -173,14 +173,14 @@ Only `qdrant_url` and `embedding` are strictly required. Everything else has sen ```python # Single-tenant prototype — implicit "default" tenant -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") # Explicit per-call (multi-tenant from a request handler) -await client.documents.ingest("F1-file.docx", collection="ir_docs", tenant_id="dev") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs", tenant_id="dev") # Bound tenant for a sequence of calls (request-scoped) tenant = client.for_tenant("dev") # XragTenant — tenant_id pre-bound -await tenant.documents.ingest("F1-file.docx", collection="ir_docs") +await tenant.documents.ingest("ir-sample-b.docx", collection="ir_docs") await tenant.retrievals.search(query="...", collection="ir_docs", top_k=10) ``` @@ -484,7 +484,7 @@ User-supplied `artifacts_dir` paths can be reused across calls — that's the wh "hostname": "ingest-worker-3", "started_at": "2026-04-27T08:14:22.481Z", "document_id": "doc_01HXY...", - "source_name": "F1-file.docx", + "source_name": "ir-sample-b.docx", "xrag_version": "0.2.0" } ``` @@ -665,7 +665,7 @@ client = Xrag( ```python # ingest as usual await client.documents.ingest( - "F1-file.docx", collection="ir_docs", tenant_id="dev", + "ir-sample-b.docx", collection="ir_docs", tenant_id="dev", ) # retrieve, then generate with whatever you want @@ -699,7 +699,7 @@ What the caller controls themselves: prompt, model choice, temperature, structur from xrag import XragSync client = XragSync.from_env() -doc = client.documents.ingest("F1-file.docx", collection="ir_docs", tenant_id="dev") +doc = client.documents.ingest("ir-sample-b.docx", collection="ir_docs", tenant_id="dev") print(client.rag.ask(query="...", collection="ir_docs", tenant_id="dev").answer) ``` @@ -936,7 +936,7 @@ query_result = run_rag("What's the revenue?", cfg) from xrag import Xrag client = Xrag.from_env() -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") result = await client.rag.ask(query="What's the revenue?", collection="ir_docs") ``` @@ -945,7 +945,7 @@ result = await client.rag.ask(query="What's the revenue?", collection="ir_docs") from xrag import Xrag client = Xrag(qdrant_url="http://qdrant:6333", embedding="openai:text-embedding-3-small") -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") search = await client.retrievals.search( query="What's the revenue?", collection="ir_docs", top_k=10, ) diff --git a/docs/survey/claude-rag-survey.md b/docs/survey/claude-rag-survey.md index d3680a8..7c755f2 100644 --- a/docs/survey/claude-rag-survey.md +++ b/docs/survey/claude-rag-survey.md @@ -11,7 +11,7 @@ Building a RAG system for financial reports (DOCX/PDF) with dense text + many ta ## CHALLENGE 1: Document Parsing & Table Extraction -**Problem**: Financial reports like the Neksai Inc. filing contain 20+ complex tables (balance sheets, income statements, cash flows, footnote tables) interleaved with dense narrative text. Standard text extraction destroys table structure, making retrieval of exact numbers impossible. +**Problem**: Financial reports like the Contoso Inc. filing contain 20+ complex tables (balance sheets, income statements, cash flows, footnote tables) interleaved with dense narrative text. Standard text extraction destroys table structure, making retrieval of exact numbers impossible. ### Solution A: MinerU + HTML Table Preservation @@ -51,7 +51,7 @@ Step 1.6 Validate extraction against golden samples: ## CHALLENGE 2: Table-Aware Chunking Strategy -**Problem**: Standard fixed-size chunking (400–512 tokens) splits tables mid-row, destroying the relationship between row headers and values. A question like "What was Neksai's total current assets in 2024?" requires the entire balance sheet table to be in one chunk, but that table may be 800+ tokens. +**Problem**: Standard fixed-size chunking (400–512 tokens) splits tables mid-row, destroying the relationship between row headers and values. A question like "What was Contoso's total current assets in 2024?" requires the entire balance sheet table to be in one chunk, but that table may be 800+ tokens. ### Solution A: Structure-Aware Chunking with Table Atomization @@ -76,9 +76,9 @@ Chunk types: ``` Parent: Full "Balance Sheet" section (table + surrounding text) - └─ Child: "Total current assets: $13,387,879 (Dec 31, 2024)" - └─ Child: "Cash: $7,351,900 (Dec 31, 2024)" - └─ Child: "Digital assets: $2,853,590 (Dec 31, 2024)" + └─ Child: "Total current assets: $15,640,290 (Dec 31, 2024)" + └─ Child: "Cash: $6,120,400 (Dec 31, 2024)" + └─ Child: "Digital assets: $3,106,470 (Dec 31, 2024)" ``` - **Pros**: Precise retrieval + complete context, solves the fundamental chunk-size tradeoff (RAGFlow's "structural conflict"), enables multi-hop reasoning @@ -99,11 +99,11 @@ Step 2.1 Define chunk hierarchy: Step 2.2 For each table, generate BOTH: a) Full table as HTML (parent chunk) b) Row-group summaries as natural language (child chunks): - "As of December 31, 2024, Neksai's cash was $7,351,900 and - digital assets were $2,853,590" + "As of December 31, 2024, Contoso's cash was $6,120,400 and + digital assets were $3,106,470" Step 2.3 Add contextual metadata to every chunk: { - "doc_id": "neksai_10k_2024", + "doc_id": "contoso_10k_2024", "section": "Consolidated Balance Sheets", "page": 3, "chunk_type": "table", @@ -120,7 +120,7 @@ Step 2.5 Validate: for each gold Q&A pair, verify the answer exists in at ## CHALLENGE 3: Hybrid Retrieval + Reranking -**Problem**: Dense vector search alone misses exact financial terms (e.g., "PCAOB ID:1171", "ASU 2023-08", specific dollar amounts). Sparse search alone misses semantic paraphrases ("profit" vs. "net income"). Financial queries require both. +**Problem**: Dense vector search alone misses exact financial terms (e.g., "PCAOB ID:9999", "ASU 2023-08", specific dollar amounts). Sparse search alone misses semantic paraphrases ("profit" vs. "net income"). Financial queries require both. ### Solution A: BM25 + Dense Vectors + Reciprocal Rank Fusion (RRF) @@ -234,10 +234,10 @@ Step 4.1 Design generation prompt with: - Uncertainty handling ("INSUFFICIENT_CONTEXT") - Output format: {answer, citations[], confidence, reasoning} Step 4.2 Implement answer type detection: - - EXACT_VALUE: "What was total revenue in 2024?" → $45,042,408 + - EXACT_VALUE: "What was total revenue in 2024?" → $48,715,220 - COMPARISON: "How did revenue change YoY?" → calculation + values - - EXPLANATION: "What is Neksai's revenue recognition policy?" → text - - YES_NO: "Does Neksai have deferred tax assets?" → Yes/No + evidence + - EXPLANATION: "What is Contoso's revenue recognition policy?" → text + - YES_NO: "Does Contoso have deferred tax assets?" → Yes/No + evidence Step 4.3 For EXACT_VALUE questions: - Extract the number from the answer - Regex-validate it exists in a retrieved chunk @@ -351,19 +351,19 @@ Evaluation layers: ### Implementation Steps ``` -Step 6.1 Build gold QA dataset from the Neksai financial statements: +Step 6.1 Build gold QA dataset from the Contoso financial statements: Category 1 — Exact Value (deterministic): - Q: "What was Neksai's net income for 2024?" - A: "$5,741,797" + Q: "What was Contoso's net income for 2024?" + A: "$6,003,510" Gold chunks: [balance_sheet_2024, income_statement_2024] Category 2 — Calculation: Q: "What was the gross profit margin in 2024?" - A: "64.1% ($28,851,979 / $45,042,408)" + A: "64.1% ($31,402,650 / $48,715,220)" Gold chunks: [income_statement_2024] Category 3 — Explanation (LLM-judged): - Q: "How does Neksai recognize revenue from subscription services?" + Q: "How does Contoso recognize revenue from subscription services?" A: "Over time within the subscription period..." Gold chunks: [note_2_revenue_recognition] @@ -374,7 +374,7 @@ Step 6.1 Build gold QA dataset from the Neksai financial statements: Gold chunks: [note_1_reorganization] Category 5 — Unanswerable: - Q: "What is Neksai's projected revenue for 2025?" + Q: "What is Contoso's projected revenue for 2025?" A: "INSUFFICIENT_CONTEXT" Gold chunks: [] (nothing should be relevant) @@ -488,7 +488,7 @@ Step 6.5 Private-set robustness measures: ### Phase 1: Foundation (Weeks 1-2) | Step | Task | Deliverable | |------|------|-------------| -| 1.1 | Set up MinerU + Docling parsing | Parsed Neksai report with 95%+ table accuracy | +| 1.1 | Set up MinerU + Docling parsing | Parsed Contoso report with 95%+ table accuracy | | 1.2 | Build multi-granularity chunker | Parent/child chunk pairs with metadata | | 1.3 | Set up vector store + BM25 index | Indexed chunks, searchable | | 1.4 | Basic retrieval pipeline (dense only) | Baseline Recall@5 measurement | diff --git a/docs/survey/datasets.md b/docs/survey/datasets.md index ca3ced8..41a6384 100644 --- a/docs/survey/datasets.md +++ b/docs/survey/datasets.md @@ -41,6 +41,6 @@ Here's the landscape of RAG evaluation datasets, organized from most to least re **FiQA** (2018) — opinion-based financial QA, mainly sentiment. **ConvFinQA** (2022) — conversational multi-turn financial QA. **HotpotQA** — general multi-hop, not financial. **NaturalQuestions** — general single-hop from Google search. -## What to use for your Neksai project +## What to use for your Contoso project Start with **FinanceBench** as your primary reference for format and question style — its evidence provenance schema (evidence_text + page + document) directly inspired your golden dataset design. Use **T2-RAGBench** or **FinQA** as a secondary reference for table-heavy numerical reasoning evaluation. For multi-hop testing, look at **FinDER** for realistic analyst-style queries. Your custom dataset fills the gap that none of these cover: single-company, full-report RAG with deterministic + LLM-judge hybrid evaluation and mandatory telemetry scoring. diff --git a/docs/survey/preprocessing-and-normalization.md b/docs/survey/preprocessing-and-normalization.md index 283acb8..2872e42 100644 --- a/docs/survey/preprocessing-and-normalization.md +++ b/docs/survey/preprocessing-and-normalization.md @@ -130,8 +130,8 @@ The largest ROI lever for financial documents. headers ("Year ended December 31" spanning 2024/2023). - **Empty row / column trimming** — visual spacers between sections produce all-empty rows; drop them. -- **Currency-cell merging** — accounting style `$ | 4,431,280` should - become a single cell `$4,431,280`; otherwise the `$` becomes its own +- **Currency-cell merging** — accounting style `$ | 3,845,200` should + become a single cell `$3,845,200`; otherwise the `$` becomes its own noise column. (This repo's `merge_table_currency_cells` step.) - **Section-row preservation** — `ASSETS`, `Current assets:` headers should remain as anchor rows, not be dropped as "empty after the @@ -328,7 +328,7 @@ Specific failure modes preprocessing should improve: because heading was a `UncategorizedText`. Caught by recall drop on questions whose answers live in body content but are heading-disambiguated. -- **Table currency mis-segmentation** — `$ | 4,431,280` → two cells → +- **Table currency mis-segmentation** — `$ | 3,845,200` → two cells → retriever returns either the `$` cell or the number cell; generator hallucinates the missing half. Caught by num_match drops on numeric-answer questions. diff --git a/examples/01_quickstart.py b/examples/01_quickstart.py index 3551b54..c2c03ad 100644 --- a/examples/01_quickstart.py +++ b/examples/01_quickstart.py @@ -15,7 +15,7 @@ from xrag import Xrag -DOC_PATH = Path("data/raw/ir_document/F1-file.docx") +DOC_PATH = Path("data/raw/ir_document/ir-sample-b.docx") COLLECTION = "ir_quickstart" diff --git a/examples/04_multi_tenant.py b/examples/04_multi_tenant.py index 1154af7..235c6aa 100644 --- a/examples/04_multi_tenant.py +++ b/examples/04_multi_tenant.py @@ -11,8 +11,8 @@ from xrag import Xrag -DOC_A = Path("data/raw/ir_document/F1-file.docx") -DOC_B = Path("data/raw/ir_document/Pj_AVATAR_F_page_260205.docx") +DOC_A = Path("data/raw/ir_document/ir-sample-b.docx") +DOC_B = Path("data/raw/ir_document/ir-sample-a.docx") COLLECTION = "ir_multitenant" diff --git a/examples/08_per_call_overrides.py b/examples/08_per_call_overrides.py index b421323..4bb66b7 100644 --- a/examples/08_per_call_overrides.py +++ b/examples/08_per_call_overrides.py @@ -12,7 +12,7 @@ from xrag import Xrag -DOC_PATH = Path("data/raw/ir_document/F1-file.docx") +DOC_PATH = Path("data/raw/ir_document/ir-sample-b.docx") COLLECTION = "ir_overrides" diff --git a/examples/09_parse_docx_api_vs_local.py b/examples/09_parse_docx_api_vs_local.py index 9b8a9c4..5394e50 100644 --- a/examples/09_parse_docx_api_vs_local.py +++ b/examples/09_parse_docx_api_vs_local.py @@ -25,7 +25,7 @@ from xrag.config.models import AppConfig, PipelineConfig, ProviderConfig, SourceConfig from xrag.pipelines.parser import run_parser_pipeline -DOC_PATH = Path("data/raw/ir_document/F1-file.docx") +DOC_PATH = Path("data/raw/ir_document/ir-sample-b.docx") ROOT = Path("data/dev/xrag_examples/09_parse_docx_api_vs_local") diff --git a/scripts/ingest/ir_latest.sh b/scripts/ingest/ir_latest.sh index 0a8f158..60abc4d 100755 --- a/scripts/ingest/ir_latest.sh +++ b/scripts/ingest/ir_latest.sh @@ -17,10 +17,10 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$REPO_ROOT" CONFIG="configs/rag-ir-latest.yml" -RAW_DIR="/data/lucas/projects/rag/ir_v2/data/dev/ir_document/raw" -OUTPUT_DIR="/data/lucas/projects/rag/ir_v2/data/dev/ir_document/latest" -F1_DOC="$RAW_DIR/F1-file.docx" -PJ_DOC="$RAW_DIR/Pj_AVATAR_F_page_260205.docx" +RAW_DIR="data/dev/ir_document/raw" +OUTPUT_DIR="data/dev/ir_document/latest" +F1_DOC="$RAW_DIR/ir-sample-b.docx" +PJ_DOC="$RAW_DIR/ir-sample-a.docx" if [[ ! -f "$CONFIG" ]]; then echo "ERROR: config not found: $CONFIG" >&2 diff --git a/tests/unit/test_chunk_prepare.py b/tests/unit/test_chunk_prepare.py index 87b4722..6e88b02 100644 --- a/tests/unit/test_chunk_prepare.py +++ b/tests/unit/test_chunk_prepare.py @@ -36,13 +36,13 @@ def test_run_chunk_prepare_pipeline_builds_section_text_and_table_chunks(tmp_pat }, { "type": "Table", - "text": "Cash | $4,431,280 | $7,351,900", + "text": "Cash | $3,845,200 | $6,120,400", "element_id": "table-1", "metadata": { "page_number": 3, "text_as_html": ( - "" - "
Cash$4,431,280$7,351,900
" + "" + "
Cash$3,845,200$6,120,400
" ), }, }, @@ -76,9 +76,9 @@ def test_run_chunk_prepare_pipeline_builds_section_text_and_table_chunks(tmp_pat assert text_chunk["parent_id"] == section_chunk["id"] assert table_chunk["parent_id"] == section_chunk["id"] assert table_chunk["metadata"]["table_html"].startswith("") - assert table_chunk["metadata"]["table_text"] == "Cash | $4,431,280 | $7,351,900" + assert table_chunk["metadata"]["table_text"] == "Cash | $3,845,200 | $6,120,400" assert table_chunk["metadata"]["table_text_markdown"] == ( - "| Cash | $4,431,280 | $7,351,900 |\n| --- | --- | --- |" + "| Cash | $3,845,200 | $6,120,400 |\n| --- | --- | --- |" ) assert ( "The balance sheets present the financial position." in table_chunk["metadata"]["pre_text"] @@ -153,13 +153,13 @@ def test_run_chunk_prepare_pipeline_supports_section_token_strategy(tmp_path: Pa }, { "type": "Table", - "text": "Cash | $4,431,280 | $7,351,900", + "text": "Cash | $3,845,200 | $6,120,400", "element_id": "table-1", "metadata": { "page_number": 3, "text_as_html": ( - "
" - "
Cash$4,431,280$7,351,900
" + "" + "
Cash$3,845,200$6,120,400
" ), }, }, @@ -196,7 +196,7 @@ def test_run_chunk_prepare_pipeline_supports_section_token_strategy(tmp_path: Pa assert table_chunk["metadata"]["chunk_strategy"] == "section_token" assert table_chunk["parent_id"] == section_chunk["id"] assert table_chunk["metadata"]["table_text_markdown"].startswith( - "| Cash | $4,431,280 | $7,351,900 |" + "| Cash | $3,845,200 | $6,120,400 |" ) assert table_chunk["metadata"]["pre_text"] assert table_chunk["metadata"]["post_text"] @@ -323,13 +323,13 @@ def test_section_table_chunker_skips_parenthetical_caption_as_heading( }, { "type": "Table", - "text": "Cash | 4,431,280 | 7,351,900", + "text": "Cash | 3,845,200 | 6,120,400", "element_id": "table-1", "metadata": { "page_number": 3, "text_as_html": ( - "" - "
Cash4,431,2807,351,900
" + "" + "
Cash3,845,2006,120,400
" ), }, }, diff --git a/tests/unit/test_embed_text.py b/tests/unit/test_embed_text.py index 3ad4e8d..c73593c 100644 --- a/tests/unit/test_embed_text.py +++ b/tests/unit/test_embed_text.py @@ -89,7 +89,7 @@ def test_table_chunk_with_summary_and_questions_orders_table_text_first() -> Non """ chunk = { "chunk_type": "table", - "table_text": "Cash | $4,431,280 | $7,351,900", + "table_text": "Cash | $3,845,200 | $6,120,400", "summary_kwd": "Balance sheet snapshot of cash position year over year.", "question_kwd": [ "What was Cash on the December 2025 balance sheet?", @@ -99,7 +99,7 @@ def test_table_chunk_with_summary_and_questions_orders_table_text_first() -> Non } assert compose_embed_text(chunk) == ( - "Cash | $4,431,280 | $7,351,900\n\n" + "Cash | $3,845,200 | $6,120,400\n\n" "Balance sheet snapshot of cash position year over year.\n\n" "Questions:\n" "What was Cash on the December 2025 balance sheet?\n" diff --git a/tests/unit/test_enrichment.py b/tests/unit/test_enrichment.py index 79c072c..69cb8da 100644 --- a/tests/unit/test_enrichment.py +++ b/tests/unit/test_enrichment.py @@ -53,8 +53,8 @@ def test_run_enrichment_pipeline_applies_ragflow_fields(monkeypatch) -> None: "chunk_id": "c2", "chunk_type": "table", "heading_path": ["CONSOLIDATED BALANCE SHEETS"], - "content": "Cash 4,431,280 7,351,900", - "table_text": "Cash | 4,431,280 | 7,351,900", + "content": "Cash 3,845,200 6,120,400", + "table_text": "Cash | 3,845,200 | 6,120,400", }, { "chunk_id": "c3", diff --git a/tests/unit/test_eval_create_dataset.py b/tests/unit/test_eval_create_dataset.py index 720652e..f281896 100644 --- a/tests/unit/test_eval_create_dataset.py +++ b/tests/unit/test_eval_create_dataset.py @@ -18,8 +18,8 @@ _SIMPLE_TABLE_HTML = ( "" "" - "" - "" + "" + "" "
December 31, 2023December 31, 2024
Cash$4,431,280$7,351,900
Digital assets$317,662$1,012,543
Cash$3,845,200$6,120,400
Digital assets$352,940$1,012,543
" ) @@ -42,7 +42,7 @@ def _build_table_chunk( "page_number": page_number, "parent_chunk_id": "test_doc:chunk:1", "table_html": table_html, - "table_text": "Cash | $4,431,280 | $7,351,900", + "table_text": "Cash | $3,845,200 | $6,120,400", "pre_text": "The balance sheets present the financial position.", "post_text": "Cash increased year over year.", } @@ -167,7 +167,7 @@ def test_parse_table_html_extracts_grid() -> None: assert "Cash" in grid.row_labels assert "Digital assets" in grid.row_labels assert len(grid.values) == 2 - assert any("4,431,280" in v for v in grid.values[0]) + assert any("3,845,200" in v for v in grid.values[0]) def test_parse_table_html_returns_none_for_empty() -> None: diff --git a/tests/unit/test_eval_dataset_review.py b/tests/unit/test_eval_dataset_review.py index a22d4a8..1bad6f4 100644 --- a/tests/unit/test_eval_dataset_review.py +++ b/tests/unit/test_eval_dataset_review.py @@ -19,7 +19,7 @@ def _sample_items() -> list[dict]: { "example_id": "doc:qa:0001", "question": "What was total cash as of Dec 31?", - "answer": "$7,351,900", + "answer": "$6,120,400", "answer_type": "numeric", "reasoning_type": "table_lookup", "difficulty": "easy", @@ -123,7 +123,7 @@ def test_load_chunks_index_handles_canonical_envelope(tmp_path: Path) -> None: { "id": "doc:chunk:12", "doc_id": "doc", - "text": "Cash at Dec 31 was 7351900.", + "text": "Cash at Dec 31 was 6120400.", "metadata": { "heading_path": ["Balance Sheet"], }, @@ -138,7 +138,7 @@ def test_load_chunks_index_handles_canonical_envelope(tmp_path: Path) -> None: index = load_chunks_index(chunks_path) - assert index["doc:chunk:12"]["content"] == "Cash at Dec 31 was 7351900." + assert index["doc:chunk:12"]["content"] == "Cash at Dec 31 was 6120400." assert index["doc:chunk:12"]["heading_path"] == ["Balance Sheet"] @@ -165,7 +165,7 @@ def test_generate_dataset_review_html_with_chunks_resolves_evidence() -> None: chunks_by_id = { "doc:chunk:12": { "chunk_id": "doc:chunk:12", - "content": "Cash at Dec 31 was 7351900.", + "content": "Cash at Dec 31 was 6120400.", "heading_path": ["Balance Sheet"], "chunk_type": "table", } @@ -173,7 +173,7 @@ def test_generate_dataset_review_html_with_chunks_resolves_evidence() -> None: html_out = generate_dataset_review_html(items, chunks_by_id=chunks_by_id) # The chunk content must be embedded somewhere so the JS can resolve it - assert "Cash at Dec 31 was 7351900." in html_out + assert "Cash at Dec 31 was 6120400." in html_out assert "Balance Sheet" in html_out @@ -207,7 +207,7 @@ def test_eval_review_cli_with_chunks(tmp_path: Path) -> None: "chunks": [ { "chunk_id": "doc:chunk:12", - "content": "Cash at Dec 31 was 7351900.", + "content": "Cash at Dec 31 was 6120400.", "heading_path": ["Balance Sheet"], "chunk_type": "table", } @@ -234,4 +234,4 @@ def test_eval_review_cli_with_chunks(tmp_path: Path) -> None: assert result.exit_code == 0, result.stdout rendered = output_path.read_text() - assert "Cash at Dec 31 was 7351900." in rendered + assert "Cash at Dec 31 was 6120400." in rendered diff --git a/tests/unit/test_eval_run.py b/tests/unit/test_eval_run.py index 7fee505..93aab0a 100644 --- a/tests/unit/test_eval_run.py +++ b/tests/unit/test_eval_run.py @@ -126,7 +126,7 @@ def _build_testset_jsonl(tmp_path: Path) -> Path: "example_id": "doc1:qa:0001", "doc_id": "doc1", "question": "What was Cash as of 2024?", - "answer": "$7,351,900", + "answer": "$6,120,400", "answer_type": "numeric", "reasoning_type": "single_hop_table", "difficulty": "easy", @@ -226,10 +226,10 @@ def test_deterministic_metrics_numeric_match() -> None: q = EvalQuestion( question_id="q1", question="Q?", - ground_truth="$4,431,280", + ground_truth="$3,845,200", answer_type="numeric", ) - metrics = _compute_deterministic_metrics(q, "The value was 4431280 dollars.", []) + metrics = _compute_deterministic_metrics(q, "The value was 3845200 dollars.", []) assert metrics["numeric_match"] == 1.0 @@ -248,7 +248,7 @@ def test_deterministic_metrics_no_evidence_skips_retrieval() -> None: def test_extract_number_handles_formatting() -> None: """Extract number from various text formats.""" - assert _extract_number("$4,431,280") == 4431280.0 + assert _extract_number("$3,845,200") == 3845200.0 assert _extract_number("about 3.5 million") == 3.5 assert _extract_number("no number here") is None diff --git a/tests/unit/test_llm_dataset_generator.py b/tests/unit/test_llm_dataset_generator.py index f78769a..0fa80a6 100644 --- a/tests/unit/test_llm_dataset_generator.py +++ b/tests/unit/test_llm_dataset_generator.py @@ -67,13 +67,13 @@ def _make_table_chunk(chunk_id: str = "test_doc:chunk:10") -> dict[str, Any]: return _make_chunk( chunk_id=chunk_id, chunk_type="table", - content="Cash | $4,431,280 | $7,351,900", + content="Cash | $3,845,200 | $6,120,400", heading_path=["CONSOLIDATED BALANCE SHEETS"], table_html=( "" "" - "" - "" + "" + "" "
December 31, 2023December 31, 2024
Cash$4,431,280$7,351,900
Digital assets$317,662$1,012,543
Cash$3,845,200$6,120,400
Digital assets$352,940$1,012,543
" ), ) @@ -150,21 +150,21 @@ def _mock_direct_lookup_response(chunk_ids: list[str]) -> DirectLookupResponse: def _mock_table_lookup_response(chunk_id: str) -> DirectLookupResponse: """Canned response for table_lookup — answers verbatim from the test - table chunk's `table_html` ($4,431,280 and $7,351,900).""" + table chunk's `table_html` ($3,845,200 and $6,120,400).""" return DirectLookupResponse( questions=[ GeneratedQA( - verbatim_span="$4,431,280", + verbatim_span="$3,845,200", question="What was cash as of December 31, 2023?", - answer="$4,431,280", + answer="$3,845,200", answer_type="numeric", difficulty="easy", evidence_chunk_id=chunk_id, ), GeneratedQA( - verbatim_span="$7,351,900", + verbatim_span="$6,120,400", question="What was cash as of December 31, 2024?", - answer="$7,351,900", + answer="$6,120,400", answer_type="numeric", difficulty="easy", evidence_chunk_id=chunk_id, @@ -634,24 +634,24 @@ def mock_invoke(prompt: str) -> Any: def test_answer_is_grounded_numeric_exact_match() -> None: """Verbatim numeric answer is grounded.""" - chunk = "Net cash provided by operating activities was $12,945,449 in 2023." - assert _answer_is_grounded("$12,945,449", chunk, "numeric") is True + chunk = "Net cash provided by operating activities was $14,203,880 in 2023." + assert _answer_is_grounded("$14,203,880", chunk, "numeric") is True def test_answer_is_grounded_numeric_rejects_magnitude_rounding() -> None: """The whole point of the strict check: '$13 million' is NOT grounded - in a chunk that says '$12,945,449' — they differ by far more than the + in a chunk that says '$14,203,880' — they differ by far more than the 0.5 % tolerance the validator uses.""" - chunk = "Net cash provided by operating activities was $12,945,449 in 2023." + chunk = "Net cash provided by operating activities was $14,203,880 in 2023." assert _answer_is_grounded("$13 million", chunk, "numeric") is False assert _answer_is_grounded("$1 million", chunk, "numeric") is False def test_answer_is_grounded_numeric_tolerates_minor_formatting() -> None: """Tiny formatting differences (within 0.5 %) still count as grounded.""" - chunk = "Cash 7,351,900 at end of year" - # 7,351,900 → answer of 7351900 (no commas) should still match - assert _answer_is_grounded("7351900", chunk, "numeric") is True + chunk = "Cash 6,120,400 at end of year" + # 6,120,400 → answer of 6120400 (no commas) should still match + assert _answer_is_grounded("6120400", chunk, "numeric") is True def test_answer_is_grounded_string_substring() -> None: @@ -681,7 +681,7 @@ def test_validator_rejects_ungrounded_numeric_answers( "doc_id": "test_doc", "chunk_type": "text", "chunk_strategy": "section_table", - "content": "Net cash provided by operating activities was $12,945,449 in 2023. " * 3, + "content": "Net cash provided by operating activities was $14,203,880 in 2023. " * 3, "heading_path": ["Cash flow"], "child_chunk_ids": [], "source_element_ids": ["e1"], @@ -698,9 +698,9 @@ def test_validator_rejects_ungrounded_numeric_answers( evidence_chunk_id="test_doc:chunk:1", ), GeneratedQA( - verbatim_span="$12,945,449", # verbatim — must keep + verbatim_span="$14,203,880", # verbatim — must keep question="Net cash from operations in 2023, exact?", - answer="$12,945,449", + answer="$14,203,880", answer_type="numeric", difficulty="easy", evidence_chunk_id="test_doc:chunk:1", @@ -736,7 +736,7 @@ def mock_invoke(prompt: str) -> Any: direct = [r for r in results if r["metadata"]["category"] == "direct_lookup"] assert len(direct) == 1 - assert direct[0]["answer"] == "$12,945,449" + assert direct[0]["answer"] == "$14,203,880" # No needs_review status should leak into the output assert all(r["metadata"].get("validation_status") != "needs_review" for r in results) @@ -956,12 +956,12 @@ def test_entity_audit_rejects_question_containing_corpus_entity() -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. is the principal operating entity incorporated in Nevada.", + content="Contoso Inc. is the principal operating entity incorporated in Nevada.", heading_path=["CORPORATE STRUCTURE"], ), ] - # Question that echoes the corpus entity "Neksai Inc." must fail audit - assert _entity_audit_fails("How is Neksai Inc. structured?", chunks) is True + # Question that echoes the corpus entity "Contoso Inc." must fail audit + assert _entity_audit_fails("How is Contoso Inc. structured?", chunks) is True def test_entity_audit_passes_generic_navigational_question() -> None: @@ -969,7 +969,7 @@ def test_entity_audit_passes_generic_navigational_question() -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. is the principal operating entity.", + content="Contoso Inc. is the principal operating entity.", heading_path=["CORPORATE STRUCTURE"], ), ] @@ -1002,31 +1002,31 @@ def test_entity_audit_is_case_insensitive() -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai BVI Ltd. was incorporated in the British Virgin Islands.", + content="Contoso BVI Ltd. was incorporated in the British Virgin Islands.", heading_path=["CORPORATE STRUCTURE"], ), ] - assert _entity_audit_fails("What is neksai bvi ltd.?", chunks) is True + assert _entity_audit_fails("What is contoso bvi ltd.?", chunks) is True def test_entity_audit_rejects_single_word_entity() -> None: - """First token of a multi-word entity (e.g. 'Neksai' from 'Neksai Inc.') is also audited.""" + """First token of a multi-word entity (e.g. 'Contoso' from 'Contoso Inc.') is also audited.""" chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. is the principal operating entity.", + content="Contoso Inc. is the principal operating entity.", heading_path=["CORPORATE STRUCTURE"], ), ] - # "Neksai" alone — not the full "Neksai Inc." phrase — must still be rejected. - assert _entity_audit_fails("What does Neksai do?", chunks) is True + # "Contoso" alone — not the full "Contoso Inc." phrase — must still be rejected. + assert _entity_audit_fails("What does Contoso do?", chunks) is True def test_entity_audit_scans_extra_text_fields() -> None: """Entity in table_text / pre_text / post_text is also caught.""" chunk = _make_chunk("doc:chunk:1", content="") - chunk["table_text"] = "Neksai BVI Ltd. is a wholly-owned subsidiary." - assert _entity_audit_fails("What does Neksai BVI Ltd. do?", [chunk]) is True + chunk["table_text"] = "Contoso BVI Ltd. is a wholly-owned subsidiary." + assert _entity_audit_fails("What does Contoso BVI Ltd. do?", [chunk]) is True def test_primary_leaf_is_deterministic() -> None: @@ -1058,7 +1058,7 @@ def test_navigational_prompt_omits_chunk_content(mock_settings: MagicMock) -> No chunks = [ _make_chunk( "doc:chunk:1", - content="SECRET_CONTENT: Neksai Inc. revenue was $1M", + content="SECRET_CONTENT: Contoso Inc. revenue was $1M", heading_path=["CORPORATE HISTORY AND STRUCTURE"], ), _make_chunk( @@ -1135,7 +1135,7 @@ def test_navigational_example_schema(mock_settings: MagicMock) -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="The company operates through Neksai Inc. and two BVI subsidiaries.", + content="The company operates through Contoso Inc. and two BVI subsidiaries.", heading_path=["CORPORATE HISTORY AND STRUCTURE"], ), ] @@ -1179,7 +1179,7 @@ def test_navigational_entity_leak_rejected_in_run(mock_settings: MagicMock) -> N chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. operates the principal business.", + content="Contoso Inc. operates the principal business.", heading_path=["CORPORATE STRUCTURE"], ), ] @@ -1188,7 +1188,7 @@ def mock_invoke(prompt: str) -> NavigationalResponse: return NavigationalResponse( questions=[ NavigationalQA( - question="What does Neksai Inc. do?", # entity leak — must be dropped + question="What does Contoso Inc. do?", # entity leak — must be dropped target_section="CORPORATE STRUCTURE", representative_answer="It operates the principal business.", ), @@ -1212,7 +1212,7 @@ def mock_invoke(prompt: str) -> NavigationalResponse: nav = [r for r in results if r["metadata"]["category"] == "navigational"] assert len(nav) == 1 - assert "Neksai" not in nav[0]["question"] + assert "Contoso" not in nav[0]["question"] @patch("tools.dataset_generation.llm.get_settings") @@ -1223,7 +1223,7 @@ def test_navigational_run_tops_up_after_entity_rejections(mock_settings: MagicMo chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. operates the principal business.", + content="Contoso Inc. operates the principal business.", heading_path=["CORPORATE STRUCTURE"], ), ] @@ -1233,7 +1233,7 @@ def test_navigational_run_tops_up_after_entity_rejections(mock_settings: MagicMo NavigationalResponse( questions=[ NavigationalQA( - question="What does Neksai Inc. do?", + question="What does Contoso Inc. do?", target_section="CORPORATE STRUCTURE", representative_answer="It operates the principal business.", ), diff --git a/tests/unit/test_parser_preprocess.py b/tests/unit/test_parser_preprocess.py index 717210a..92b15d2 100644 --- a/tests/unit/test_parser_preprocess.py +++ b/tests/unit/test_parser_preprocess.py @@ -32,7 +32,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N "avatar_comments_doc": [ { "type": "UncategorizedText", - "text": "NEKSAI INC.", + "text": "CONTOSO INC.", "element_id": "boilerplate-1", "metadata": { "category_depth": 0, @@ -42,7 +42,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N }, { "type": "UncategorizedText", - "text": "NEKSAI INC.", + "text": "CONTOSO INC.", "element_id": "boilerplate-2", "metadata": { "category_depth": 0, @@ -52,7 +52,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N }, { "type": "UncategorizedText", - "text": "NEKSAI INC.", + "text": "CONTOSO INC.", "element_id": "boilerplate-3", "metadata": { "category_depth": 0, @@ -96,7 +96,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N }, { "type": "Table", - "text": "Cash $ 4,431,280 $ 7,351,900", + "text": "Cash $ 3,845,200 $ 6,120,400", "element_id": "table-1", "metadata": { "filename": "avatar-with-comments.docx", @@ -104,7 +104,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N "" "" "" - "" + "" "
December 31, 2023December 31, 2023December 31, 2024December 31, 2024
Cash$4,431,280$7,351,900
Cash$3,845,200$6,120,400
" ), }, @@ -138,17 +138,17 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N table_element = next( element for element in normalized_elements if element["element_id"] == "table-1" ) - assert "$4,431,280" in table_element["text"] + assert "$3,845,200" in table_element["text"] assert "December 31, 2023" in table_element["text"] assert "December 31, 2024" in table_element["text"] review_html = Path(result.review_html_path).read_text() rendered_rows = _table_rows_from_html(review_html) - assert "$4,431,280" in review_html - assert "$7,351,900" in review_html + assert "$3,845,200" in review_html + assert "$6,120,400" in review_html assert ">$" not in review_html assert rendered_rows[0] == ["", "December 31, 2023", "", "December 31, 2024"] - assert rendered_rows[1] == ["Cash", "", "$4,431,280", "$7,351,900"] + assert rendered_rows[1] == ["Cash", "", "$3,845,200", "$6,120,400"] warning_codes = {warning.code for warning in result.warnings} assert "missing_page_numbers" in warning_codes @@ -214,14 +214,14 @@ def test_run_parser_preprocess_pipeline_can_enable_table_column_merge(tmp_path: [ { "type": "Table", - "text": "Cash $ 4,431,280 $ 7,351,900", + "text": "Cash $ 3,845,200 $ 6,120,400", "element_id": "table-1", "metadata": { "text_as_html": ( "" "" "" - "" + "" "
December 31, 2023December 31, 2023December 31, 2024December 31, 2024
Cash$4,431,280$7,351,900
Cash$3,845,200$6,120,400
" ), }, @@ -242,7 +242,7 @@ def test_run_parser_preprocess_pipeline_can_enable_table_column_merge(tmp_path: review_html = Path(result.review_html_path).read_text() rendered_rows = _table_rows_from_html(review_html) assert rendered_rows[0] == ["", "December 31, 2023", "December 31, 2024"] - assert rendered_rows[1] == ["Cash", "$4,431,280", "$7,351,900"] + assert rendered_rows[1] == ["Cash", "$3,845,200", "$6,120,400"] def test_run_parser_preprocess_pipeline_describes_images_when_enabled(tmp_path: Path) -> None: @@ -441,14 +441,14 @@ def test_parser_preprocess_command_accepts_config(tmp_path: Path) -> None: [ { "type": "Table", - "text": "Cash $ 4,431,280 $ 7,351,900", + "text": "Cash $ 3,845,200 $ 6,120,400", "element_id": "table-1", "metadata": { "text_as_html": ( "" "" "" - "" + "" "
December 31, 2023December 31, 2023December 31, 2024December 31, 2024
Cash$4,431,280$7,351,900
Cash$3,845,200$6,120,400
" ), }, diff --git a/tests/unit/test_verify_executors.py b/tests/unit/test_verify_executors.py index 3c8d5da..11ee486 100644 --- a/tests/unit/test_verify_executors.py +++ b/tests/unit/test_verify_executors.py @@ -25,7 +25,7 @@ def _write_base(tmp_path: Path) -> Path: "doc_ids": ["f1-doc", "pj-doc"], }, "sources": [ - {"path": "data/raw/ir_document/F1-file.docx", "doc_id": "f1-doc"}, + {"path": "data/raw/ir_document/ir-sample-b.docx", "doc_id": "f1-doc"}, {"path": "data/raw/ir_document/Pj.docx", "doc_id": "pj-doc"}, ], "vector_store": { @@ -62,7 +62,7 @@ def test_derive_config_rewrites_pipeline_and_paths(tmp_path: Path) -> None: assert derived["pipeline"]["artifact_dir"] == str(artifact_dir) # Sources keep their doc_id, paths rewritten under raw_dir by basename. assert derived["sources"] == [ - {"path": str(raw_dir / "F1-file.docx"), "doc_id": "f1-doc"}, + {"path": str(raw_dir / "ir-sample-b.docx"), "doc_id": "f1-doc"}, {"path": str(raw_dir / "Pj.docx"), "doc_id": "pj-doc"}, ] assert derived["vector_store"]["options"]["path"] == str(qdrant_path) diff --git a/tools/dataset_generation/llm.py b/tools/dataset_generation/llm.py index 7af8c7e..d1708ca 100644 --- a/tools/dataset_generation/llm.py +++ b/tools/dataset_generation/llm.py @@ -149,7 +149,7 @@ def _normalize_for_match(text: str) -> str: def _extract_numbers(text: str) -> list[float]: """Pull all numbers from a string (handles commas, decimals, magnitudes).""" out: list[float] = [] - # Find tokens like 12,945,449 or 5.9 (without magnitude) + # Find tokens like 14,203,880 or 5.9 (without magnitude) for m in re.finditer(r"-?[\d,]*\d[\d,]*\.?\d*", text or ""): token = m.group() try: @@ -166,7 +166,7 @@ def _answer_is_grounded( For numeric answers: every number in the answer must appear in the chunk (within `tol` relative tolerance — default 0.5 % to account for cell - formatting differences but reject magnitude rounding like 5.9M vs 5,898,348). + formatting differences but reject magnitude rounding like 5.9M vs 6,357,910). For string answers: the answer's normalized content must be a substring of the normalized chunk. @@ -260,9 +260,9 @@ def _build_corpus_entities(chunks: list[dict[str, Any]]) -> set[str]: """Extract a lowercase entity set from all text fields across chunks. Scans ``content``, ``table_text``, ``pre_text``, and ``post_text``. - Multi-word entity phrases (e.g. ``"neksai inc."``) are stored whole; the + Multi-word entity phrases (e.g. ``"contoso inc."``) are stored whole; the first token is also added when it has ≥5 characters and is not a known legal - suffix so that single-word usage (e.g. ``"Neksai"``) is caught by the audit. + suffix so that single-word usage (e.g. ``"Contoso"``) is caught by the audit. Args: chunks: Chunk dicts from chunks.json. @@ -667,20 +667,20 @@ def _generate_table_lookup( "3. Copy `verbatim_span` unchanged into `answer`.\n\n" "**Strict formatting rules (violations cause the row to be " "rejected):**\n" - '- NO magnitude compression: if the cell shows "$12,945,449" ' - 'the verbatim_span must be "$12,945,449", NOT "$13M" or ' + '- NO magnitude compression: if the cell shows "$14,203,880" ' + 'the verbatim_span must be "$14,203,880", NOT "$13M" or ' '"about $13 million".\n' "- NO unit conversion: copy the number at the scale it appears.\n" '- NO paraphrasing percentages: copy "33.7%" not "~34%".\n' "- Preserve the source's sign convention (parentheses for " "negatives if the table uses them).\n" "- Do NOT append unit words to numbers " - '("$7,351,900" not "$7,351,900 dollars").\n\n' + '("$6,120,400" not "$6,120,400 dollars").\n\n' "**Few-shot examples:**\n\n" "GOOD:\n" - ' verbatim_span: "$4,431,280"\n' + ' verbatim_span: "$3,845,200"\n' ' question: "What was the cash balance as of December 31, 2023?"\n' - ' answer: "$4,431,280"\n' + ' answer: "$3,845,200"\n' " answer_type: numeric\n\n" "GOOD:\n" ' verbatim_span: "33.7%"\n' @@ -689,9 +689,9 @@ def _generate_table_lookup( ' answer: "33.7%"\n' " answer_type: numeric\n\n" "BAD — never do this:\n" - ' verbatim_span: "about $4.4 million" ← compressed; not in ' + ' verbatim_span: "about $3.8 million" ← compressed; not in ' "the table\n" - ' answer: "$4.4M" ← also wrong\n\n' + ' answer: "$3.8M" ← also wrong\n\n' "Other rules:\n" "- Each question must be answerable from exactly ONE table chunk.\n" "- The question MUST name the exact time period from the column " @@ -818,20 +818,20 @@ def _call_direct_lookup(self, chunks: list[dict[str, Any]], count: int) -> list[ "If you cannot find a suitable verbatim span, skip that fact and " "pick a different one.\n\n" "**Strict formatting rules (violations cause rejection):**\n" - '- NO magnitude compression: "$12,945,449" not "$13M" or ' + '- NO magnitude compression: "$14,203,880" not "$13M" or ' '"~$13 million".\n' "- NO paraphrasing: copy names, dates, percentages exactly as " "written in the chunk.\n" - '- NO unit words appended to numbers: "$7,351,900" not ' - '"$7,351,900 dollars".\n' + '- NO unit words appended to numbers: "$6,120,400" not ' + '"$6,120,400 dollars".\n' "- For string answers, use the shortest verbatim span — no " "surrounding narrative.\n\n" "**Few-shot examples:**\n\n" "GOOD:\n" - ' verbatim_span: "$12,945,449"\n' + ' verbatim_span: "$14,203,880"\n' ' question: "What was net cash provided by operating activities' ' for the year ended December 31, 2023?"\n' - ' answer: "$12,945,449"\n' + ' answer: "$14,203,880"\n' " answer_type: numeric\n\n" "GOOD:\n" ' verbatim_span: "AI Avatar Co., Ltd"\n' diff --git a/tools/dataset_generation/template.py b/tools/dataset_generation/template.py index 7b6fc76..d9079e0 100644 --- a/tools/dataset_generation/template.py +++ b/tools/dataset_generation/template.py @@ -216,7 +216,7 @@ def _parse_numeric(text: str) -> float | None: """Parse a financial numeric string into a float. Args: - text: Raw cell value like "$4,431,280" or "(317,662)". + text: Raw cell value like "$3,845,200" or "(352,940)". Returns: Parsed float value, or None if unparseable. diff --git a/xrag/core/generation/openai.py b/xrag/core/generation/openai.py index 54c6e99..0f23d4a 100644 --- a/xrag/core/generation/openai.py +++ b/xrag/core/generation/openai.py @@ -19,7 +19,7 @@ "Numeric extraction rules — when the answer is a number from a table:\n" " 1. Return the EXACT value as it appears in the source, including " "every digit and the original formatting " - '(e.g. "$12,945,449", not "$13M" or "~$13 million"). Do NOT ' + '(e.g. "$14,203,880", not "$13M" or "~$13 million"). Do NOT ' "round, approximate, or compress to a magnitude word.\n" " 2. If multiple cells could match, pick the ONE whose row label " "AND column header EXACTLY match the question's metric name and " @@ -40,8 +40,8 @@ "#3 contains it. When in doubt, attempt to extract.\n" "\n" "When a calculation is required, show the component values from " - "the context first (e.g., 'Revenue 2024 = $45,042,408, Revenue " - "2023 = $23,761,586'), then state the computed result. Never " + "the context first (e.g., 'Revenue 2024 = $48,715,220, Revenue " + "2023 = $26,118,340'), then state the computed result. Never " "invent values not present in the context. Be concise." ) @@ -78,8 +78,8 @@ def _chunks_to_documents(chunks: list[dict[str, Any]]) -> list[Document]: back to markdown. TODO: improve ``src/core/common/converter.py:table_html_to_markdown`` - to drop empty pad columns, merge currency cells (``$ | 4,431,280`` - → ``$4,431,280``), and collapse duplicate multi-row date headers. + to drop empty pad columns, merge currency cells (``$ | 3,845,200`` + → ``$3,845,200``), and collapse duplicate multi-row date headers. Then flip the priority above so markdown wins by default and HTML is the fallback. """ From 3c26ca0adb356f677565b66c8403c9f2a667062b Mon Sep 17 00:00:00 2001 From: henryle97 Date: Fri, 7 Aug 2026 14:04:20 +0700 Subject: [PATCH 2/5] docs: add contributing, security, conduct, and changelog docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem. The repo had no contributor-facing docs. The PR template cited docs/contributing.md §3 and §3.1, a file that never existed. The README advertised configs/chunker.yml and configs/chunker-ragflow.yml, neither of which exists, linked to the gitignored data/ directory, and carried a phasing table claiming v0.1 was current while the project sits at v0.5.3 — the same README both used the Xrag client in its quickstart and said that client "ships in v0.2". What changed. CONTRIBUTING.md is numbered so the PR template's §3 and §3.1 references resolve, and it records the setup, checks, commit conventions, and eval discipline the project already follows. SECURITY.md routes vulnerability reports to private advisories and documents scope and credential handling. CODE_OF_CONDUCT.md is Contributor Covenant 2.1. CHANGELOG.md is reconstructed from the existing tags. The README's dead config links now point at configs/chunkers/, the data/ link is prose, and the stale phasing table is replaced by a status section describing what ships today versus what is planned. What did NOT change. No behavior, no public API. Code of Conduct enforcement routes through GitHub private advisories rather than an email address, since publishing a personal address is the maintainer's call. Co-Authored-By: Claude Opus 5 (1M context) --- .github/pull_request_template.md | 4 +- CHANGELOG.md | 106 ++++++++++++++++++++++ CODE_OF_CONDUCT.md | 126 ++++++++++++++++++++++++++ CONTRIBUTING.md | 150 +++++++++++++++++++++++++++++++ README.md | 54 ++++++++--- SECURITY.md | 52 +++++++++++ 6 files changed, 476 insertions(+), 16 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ce1fed4..9faf2a8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,5 @@ @@ -24,7 +24,7 @@ ONE `Closes #N` per issue, each on its own content line. GitHub's auto-close parser only fires on `Closes`/`Fixes`/`Resolves` when the keyword is on the same content line as the reference — a heading like `## Closes` followed by `#19, #20, #21` will silently -fail to close anything. See docs/contributing.md §3.1. +fail to close anything. See CONTRIBUTING.md §3.1. Single-issue example: Closes #42 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..02a141d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,106 @@ +# Changelog + +All notable changes to this project are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Only names exported from `xrag.__all__` are covered by the version contract. +Anything outside that surface is internal and may change in any release. + +## [Unreleased] + +### Added + +- DOCX image extraction: `DocxPicturePartitioner` and an `extract_images` + parser option (#61) +- Image chunks in `SectionTableChunker` (#62) +- LLM-generated image descriptions (#64) + +### Changed + +- Aligned retrieval and indexing defaults (#67) + +## [0.5.3] — 2026-05-05 + +### Added + +- Navigational testset generator, with ADR-0003 recording the primary-chunk + evidence decision (#51) +- Navigation questions in dataset generation (#54) +- Per-category metric breakdown by `reasoning_type` (#52) +- Append mode for the `eval create-dataset` CLI (#53) + +### Changed + +- Indexing now augments retrieval text instead of replacing it, per ADR-0002 + (#45) + +## [0.5.2] — 2026-05-05 + +### Fixed + +- Pin `openai_api_type=None` in the embedder to stop an environment variable + leaking into requests (#42) + +## [0.5.1] — 2026-05-05 + +### Added + +- `ChunkMetadata` promoted to the public `__all__` surface (#40) + +## [0.5.0] — 2026-05-04 + +### Added + +- Per-step `max_concurrency` for LLM-bound enrichers (#39) + +### Changed + +- **Breaking:** unified the chunk schema on the canonical V2 envelope and + `doc_id` (#37) + +## [0.4.1] — 2026-05-04 + +### Added + +- `qdrant_api_key` parameter on `Xrag.__init__` (#29) + +## [0.4.0] — 2026-05-03 + +### Changed + +- Explicit settings injection across the asymmetric boundary, recorded in + ADR-0001 (#26) + +## [0.3.0] — 2026-05-01 + +### Changed + +- Library and client SDK cleanup pass across the public surface (#3) + +## [0.2.0] — 2026-04-27 + +### Added + +- High-level async `Xrag` client with resource-namespaced operations + (`documents.*`, `retrievals.*`, `rag.*`) +- Tenant binding via `client.for_tenant(...)` +- Typed error hierarchy rooted at `XragError` +- Split the local-PDF parser into its own `parser-unstructured-local-pdf` extra + +### Changed + +- Packaging moved to the hatch backend; the project publishes to PyPI as + `pyxrag` while the import name stays `xrag` +- Renamed the engine repository from `fin-rag` to `xrag` + +[Unreleased]: https://github.com/henryle97/xrag/compare/v0.5.3...HEAD +[0.5.3]: https://github.com/henryle97/xrag/compare/v0.5.2...v0.5.3 +[0.5.2]: https://github.com/henryle97/xrag/compare/v0.5.1...v0.5.2 +[0.5.1]: https://github.com/henryle97/xrag/compare/v0.5.0...v0.5.1 +[0.5.0]: https://github.com/henryle97/xrag/compare/v0.4.1...v0.5.0 +[0.4.1]: https://github.com/henryle97/xrag/compare/v0.4.0...v0.4.1 +[0.4.0]: https://github.com/henryle97/xrag/compare/v0.3.0...v0.4.0 +[0.3.0]: https://github.com/henryle97/xrag/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/henryle97/xrag/releases/tag/v0.2.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..6bb9929 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,126 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples include using an official email address, posting via an official social +media account, or acting as an appointed representative at an online or offline +event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the project maintainers by opening a +[private security advisory](https://github.com/henryle97/xrag/security/advisories/new), +which is visible only to maintainers, or by contacting +[@henryle97](https://github.com/henryle97) directly on GitHub. + +All complaints will be reviewed and investigated promptly and fairly. All +community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. Violating +these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. Violating these +terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla]. + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. + +[homepage]: https://www.contributor-covenant.org +[mozilla]: https://github.com/mozilla/diversity diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..484dcbb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,150 @@ +# Contributing to xrag + +Thanks for your interest in xrag. This document covers how to get set up, the +checks we expect before review, and the conventions the PR template encodes. + +## 1. Getting set up + +xrag uses [uv](https://docs.astral.sh/uv/) for dependency management and +requires Python 3.11 or newer. + +```bash +git clone https://github.com/henryle97/xrag.git +cd xrag +make sync # uv sync --group dev + the extras the tests need +make precommit-install # installs the pre-commit hooks (recommended) +``` + +`make sync` installs the `cli`, `chroma`, and `rerank-cohere` extras because +the unit tests import them directly. A bare `uv sync` is not enough to run the +test suite. + +Local PDF/OCR parsing is heavy and optional. Add it only if you're working on +that path: + +```bash +uv sync --group dev --extra cli --extra chroma --extra rerank-cohere \ + --extra parser-unstructured-local-pdf +``` + +### Environment variables + +Copy `.env.example` to `.env` and fill in only the keys you need. Every +variable maps to a field on `xrag.config.settings.Settings`. Unit tests are +hermetic and need no credentials. + +## 2. Development workflow + +Run these before opening a PR: + +```bash +make lint # ruff check + ruff format --check +make unit-test # pytest tests/unit -q — hermetic, this is the CI gate +``` + +`make check` additionally runs `compileall`. Integration tests +(`make integration-test`) make real network calls, are not part of CI, and +require credentials. + +After changing runtime behavior, also exercise the changed path with a live CLI +command — unit tests alone don't prove a pipeline still runs end to end. + +### Tests + +- Unit tests live in `tests/unit/` and must stay hermetic — no network, no + credentials, no reliance on files outside the repo. +- Contract tests in `tests/contract/` run against the built wheel and guard the + public API surface. If you add or remove anything in `xrag.__all__`, update + them. +- Mark network-dependent tests with `requires_network`, `requires_qdrant`, or + `requires_openai`. + +### Public API surface + +Everything exported from `xrag.__all__` is the supported surface and is +SemVer-bound. Anything outside it is internal. Changing an exported name is a +breaking change and needs a contract-test update plus a changelog entry. + +## 3. Pull requests + +The [PR template](.github/pull_request_template.md) asks for four things. +Keep the shape: + +- **Problem** — what's broken or missing +- **Why** — why it's worth changing now +- **What changed** — the actual diff, in prose +- **What did NOT change** — scope boundaries, so reviewers know where not to look + +Include a **Test plan** with `make lint` and `make unit-test` checked, plus any +live CLI, integration, or manual verification you did. Skip the Risks section +entirely when there are none rather than writing "None". + +Other expectations: + +- Branch from an up-to-date `main`: `git checkout main && git pull --ff-only origin main` +- Keep PRs focused. Parent/child issues ship together as the parent's PR. +- Resolve review threads only after the fix is pushed. +- Don't auto-merge. + +### 3.1 Closing issues + +GitHub's auto-close parser only fires when a close keyword (`Closes`, `Fixes`, +`Resolves`) sits on the same content line as the issue reference. A heading like +`## Closes` followed by `#19, #20, #21` **silently closes nothing**. + +Write one close keyword per issue, each on its own content line: + +```text +Closes #19. Closes #20. Closes #21. +``` + +Verify before merging: + +```bash +gh pr view --json closingIssuesReferences +``` + +## 4. Commit conventions + +We use [Conventional Commits](https://www.conventionalcommits.org/) with a +module scope: + +```text +feat(parser): add DocxPicturePartitioner +fix(config): reject unknown provider names +test(chunker): cover image chunk emission +docs(plans): refresh indexing backlog +``` + +Common scopes: `parser`, `chunker`, `embedder`, `indexer`, `client`, `config`, +`eval`, `ci`, `packaging`. + +## 5. Architecture decisions + +Check [`docs/adr/`](docs/adr/) before making architectural changes — it records +decisions that are easy to accidentally reverse. Add a new ADR when you change +a cross-module contract or reverse an existing decision. + +Deeper domain context lives in [`CONTEXT.md`](CONTEXT.md) and +[`docs/agents/domain.md`](docs/agents/domain.md). + +## 6. Evaluation and benchmarks + +Retrieval quality changes need evidence. The full discipline is documented in +[`CLAUDE.md`](CLAUDE.md); the short version: + +- Run evals through `scripts/eval/`, which writes to `experiments/eval/-/`. +- **Append** a new row to [`docs/BENCHMARK.md`](docs/BENCHMARK.md) — never edit + existing rows. +- Per-category `R@10` is the primary regression signal. +- `experiments/eval/experiments.jsonl` is the append-only source of truth. + +Benchmark documentation uses a fictional issuer and synthetic figures. Never +commit real customer documents, real financial data, or anything under `data/` +(it is gitignored for this reason). + +## 7. Reporting bugs and requesting features + +Use the [issue templates](https://github.com/henryle97/xrag/issues/new/choose). +For security issues, follow [SECURITY.md](SECURITY.md) instead of opening a +public issue. diff --git a/README.md b/README.md index ee6433f..e7116fa 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ Recommended path: 7. Use [`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md) for the high-level Python client design. 8. Use [`docs/plans.md`](docs/plans.md) and [`docs/sub-plans/`](docs/sub-plans/) for roadmap and implementation details. -## Quickstart — library (v0.1, functional API) +## Quickstart — library (functional API) ```python from pathlib import Path @@ -369,7 +369,11 @@ result = run_rag("What was Cash as of December 31, 2024?", cfg) print(result.answer) ``` -The high-level `Xrag` async client (`client.documents.ingest(...)`, `client.retrievals.search(...)`, `client.rag.ask(...)`) ships in v0.2 — see [`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md). +Most applications should prefer the high-level `Xrag` client shown above. Use +the functional API when you already work in terms of `AppConfig` and want +direct control over pipeline wiring — see +[`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md) for the +design behind both surfaces. ## Quickstart — CLI @@ -414,7 +418,7 @@ uv run python -m xrag.cli parser preprocess \ # Chunk uv run python -m xrag.cli chunk prepare \ --input data/dev/ir_document/normalized/elements.json \ - --config configs/chunker.yml \ + --config configs/chunkers/section-table.yml \ --output-dir data/dev/ir_document/chunked # Baseline RAG @@ -447,15 +451,25 @@ UNSTRUCTURED_API_URL=... # optional For DOCX-to-PDF conversion install LibreOffice so `soffice` / `libreoffice` is on `PATH`. -## Phasing +## Project status -| Version | Surface | -|---|---| -| **v0.1** (current) | Functional API: `run_ingest`, `run_rag`, Pydantic models, configs, loaders. | -| **v0.2** | High-level `Xrag` async client with resource-namespaced ops (`documents.*`, `retrievals.*`, `rag.*`). Tenant binding via `for_tenant`. Typed error hierarchy. | -| **v0.3** | `XragSync` mirror. | -| **v0.4** | Per-call `config_overrides`, batch ingest, persistent registry contract. | -| **v0.5+ (backlog)** | Streaming surfaces (`documents.ingest_stream`, `rag.ask_stream`). | +`xrag` is pre-1.0. Everything exported from `xrag.__all__` is the supported +surface; anything outside it is internal and not SemVer-bound. Minor releases +may still change internals. + +**Available today:** + +- Functional API — `run_ingest`, `run_rag`, Pydantic models, config loaders +- High-level async `Xrag` client with resource-namespaced ops (`documents.*`, + `retrievals.*`, `rag.*`) +- Tenant binding via `client.for_tenant(...)` +- Typed error hierarchy (`XragError` and subclasses) +- Per-call ingest and retrieval overrides + +**Planned:** + +- `XragSync` — a synchronous mirror of the async client +- Streaming surfaces (`documents.ingest_stream`, `rag.ask_stream`) Full design: [`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md). @@ -492,8 +506,8 @@ uv sync --group dev --extra cli --extra chroma --extra rerank-cohere --extra par - Parser provider: `unstructured`. - Preprocess config: [`configs/preprocess.yml`](configs/preprocess.yml). -- Default chunker: `section_table` in [`configs/chunker.yml`](configs/chunker.yml). -- Alternative chunker: `ragflow` in [`configs/chunker-ragflow.yml`](configs/chunker-ragflow.yml). +- Default chunker: `section_table` in [`configs/chunkers/section-table.yml`](configs/chunkers/section-table.yml). +- Alternative chunkers: `section_token` (alias `ragflow`) in [`configs/chunkers/section-token.yml`](configs/chunkers/section-token.yml), and `title_hierarchy` in [`configs/chunkers/title-hierarchy.yml`](configs/chunkers/title-hierarchy.yml). - Simple one-shot RAG config: [`configs/rag-simple-baseline.yml`](configs/rag-simple-baseline.yml). - Persistent index/query config: [`configs/rag-baseline.yml`](configs/rag-baseline.yml). - Enrichment scenario configs: [`configs/enrichment/`](configs/enrichment). @@ -507,7 +521,8 @@ uv sync --group dev --extra cli --extra chroma --extra rerank-cohere --extra par - [`tools/`](tools/): **dev-only** — eval scoring, dataset generation, HTML viewers, eval datasets. Excluded from the wheel. - [`configs/`](configs/): runtime YAML configs. - [`docs/`](docs/README.md): user docs, plans, surveys, and terminology. -- [`data/`](data/): local inputs and generated artifacts. +- `data/`: local inputs and generated artifacts. Gitignored — created on first + run, never committed. ## Development checks @@ -530,6 +545,17 @@ GitHub Actions now verifies the same core paths contributors should run locally: Tag pushes matching `v*` reuse the verified package artifacts and publish them to the GitHub Release instead of rebuilding a second time during release. +## Contributing + +Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, +the checks expected before review, and PR conventions. Please also read the +[Code of Conduct](CODE_OF_CONDUCT.md). + +Release history is in [CHANGELOG.md](CHANGELOG.md). + +To report a security issue, follow [SECURITY.md](SECURITY.md) rather than +opening a public issue. + ## License MIT — see [LICENSE](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2fc217f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,52 @@ +# Security Policy + +## Supported versions + +xrag is pre-1.0. Security fixes land on the latest minor release only. Please +upgrade before reporting an issue against an older version. + +| Version | Supported | +| ------- | --------- | +| 0.5.x | ✅ | +| < 0.5 | ❌ | + +## Reporting a vulnerability + +**Please do not open a public issue for security problems.** + +Report privately through GitHub's +[private vulnerability reporting](https://github.com/henryle97/xrag/security/advisories/new) +for this repository. If that is unavailable to you, open a public issue +containing only a request for a private contact channel — no details. + +Please include: + +- A description of the issue and its impact +- Steps to reproduce, or a proof-of-concept +- Affected version and relevant configuration (redact all credentials) + +You can expect an initial response within 7 days and a status update at least +every 14 days until the issue is resolved. + +## Scope + +xrag is a library that talks to third-party services (OpenAI, Cohere, Qdrant, +Unstructured, AWS Bedrock). Vulnerabilities in those services belong to their +respective vendors. In scope for this project: + +- Credential handling and leakage through logs, artifacts, traces, or errors +- Tenant isolation failures in `client.for_tenant(...)` or collection scoping +- Path traversal or unsafe file handling in the parser and artifact-store paths +- Deserialization or injection issues in config, chunk, or dataset loading +- Dependency vulnerabilities reachable through xrag's default code paths + +## Handling credentials + +xrag reads credentials from environment variables only — never from config +YAML. When filing any issue or sharing a config, redact `OPENAI_API_KEY`, +`COHERE_API_KEY`, `QDRANT_API_KEY`, `UNSTRUCTURED_API_KEY`, AWS credentials, +and tracing keys. + +This repository runs [gitleaks](https://github.com/gitleaks/gitleaks) as a +pre-commit hook. Install the hooks with `make precommit-install` so secrets are +caught before they reach a commit. From 8ffd2a705f7432668fbad0cac0a60bec7b6829ff Mon Sep 17 00:00:00 2001 From: henryle97 Date: Fri, 7 Aug 2026 14:04:21 +0700 Subject: [PATCH 3/5] fix(build): install test extras in make targets, add issue and dep automation Problem. A fresh clone could not run the test suite. make unit-test ran pytest against a bare uv sync, but tests/unit/test_vector_store.py imports chromadb, so collection failed before a single test ran. CI passed only because its unit-test job adds the extras by hand. This was the first thing a new contributor would hit. What changed. A DEV_EXTRAS variable now feeds both make sync and make unit-test, matching the CI job and commented as such so the two stay in step. Dropped a ruff per-file-ignore for news.py, a file not in the repo. Added issue templates that route security reports away from public issues, and a dependabot config that groups routine bumps; LangChain majors are ignored there because they are pinned deliberately and exercised by the contract matrix in ci.yml. What did NOT change. CI job definitions, and no runtime behavior. Co-Authored-By: Claude Opus 5 (1M context) --- .github/ISSUE_TEMPLATE/bug_report.yml | 84 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 +++ .github/ISSUE_TEMPLATE/feature_request.yml | 54 ++++++++++++++ .github/dependabot.yml | 36 ++++++++++ Makefile | 8 ++- pyproject.toml | 2 - 6 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/dependabot.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..89ae507 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,84 @@ +name: Bug report +description: Something in xrag behaves incorrectly +labels: ["needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for filing a bug. Please redact all API keys and any + confidential document content before submitting. + + For security vulnerabilities, follow + [SECURITY.md](https://github.com/henryle97/xrag/blob/main/SECURITY.md) + instead of opening a public issue. + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What did you observe, and what did you expect instead? + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: > + A minimal snippet or CLI invocation. Include the relevant config with + credentials redacted. + render: shell + validations: + required: true + + - type: input + id: version + attributes: + label: xrag version + description: "Output of `python -c \"import xrag; print(xrag.__version__)\"`" + placeholder: "0.5.3" + validations: + required: true + + - type: input + id: python-version + attributes: + label: Python version + placeholder: "3.12.9" + validations: + required: true + + - type: dropdown + id: area + attributes: + label: Which area? + options: + - parser + - chunker + - enrichment + - embedding + - indexing / vector store + - retrieval / rerank + - generation + - Xrag client + - CLI + - eval / benchmark + - packaging / install + - not sure + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Logs or traceback + description: Paste the full traceback if there is one. Redact credentials. + render: shell + + - type: textarea + id: context + attributes: + label: Anything else? + description: > + Vector store backend, provider models, document type, or anything else + that helps reproduce. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..c91fd72 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Question or usage help + url: https://github.com/henryle97/xrag/discussions + about: Ask how to use xrag, or discuss an idea before filing a feature request. + - name: Security vulnerability + url: https://github.com/henryle97/xrag/security/advisories/new + about: Report privately. Please do not open a public issue for security problems. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..dfbddc5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,54 @@ +name: Feature request +description: Suggest a capability or improvement for xrag +labels: ["needs-triage"] +body: + - type: textarea + id: problem + attributes: + label: What problem are you trying to solve? + description: > + Describe the use case rather than a specific implementation. What are + you unable to do today? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: What would the API, config, or CLI surface look like? + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Workarounds you tried, or other designs you weighed. + + - type: dropdown + id: area + attributes: + label: Which area? + options: + - parser + - chunker + - enrichment + - embedding + - indexing / vector store + - retrieval / rerank + - generation + - Xrag client + - CLI + - eval / benchmark + - packaging / install + - not sure + validations: + required: true + + - type: checkboxes + id: contribution + attributes: + label: Contribution + options: + - label: I'd be willing to open a PR for this diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e9bc56f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,36 @@ +version: 2 +updates: + # Keep CI actions current — these are the highest-value updates for a + # library, since a compromised action runs with repo credentials. + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + labels: ["needs-triage"] + commit-message: + prefix: "ci" + + # Python dependencies. Grouped so routine patch bumps arrive as one PR + # rather than flooding the queue. + - package-ecosystem: uv + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + labels: ["needs-triage"] + commit-message: + prefix: "chore(deps)" + groups: + patch-and-minor: + update-types: + - minor + - patch + ignore: + # LangChain majors are pinned deliberately and exercised by the contract + # matrix in ci.yml. Bump these by hand alongside that matrix. + - dependency-name: "langchain" + update-types: ["version-update:semver-major"] + - dependency-name: "langchain-core" + update-types: ["version-update:semver-major"] + - dependency-name: "langchain-openai" + update-types: ["version-update:semver-major"] diff --git a/Makefile b/Makefile index 404bc19..bca996c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .PHONY: sync lint format check unit-test integration-test parser parser-smoke precommit-install precommit-run +# Extras the unit tests import directly (chromadb, cohere, typer). Kept in sync +# with the `unit-test` job in .github/workflows/ci.yml. +DEV_EXTRAS := --extra cli --extra chroma --extra rerank-cohere + sync: - uv sync + uv sync --group dev $(DEV_EXTRAS) lint: uv run ruff check . @@ -14,7 +18,7 @@ check: lint python3 -m compileall xrag examples tools unit-test: - uv run pytest tests/unit -q + uv run $(DEV_EXTRAS) pytest tests/unit -q integration-test: uv run pytest tests/integration -q diff --git a/pyproject.toml b/pyproject.toml index e616dc9..d792d28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,8 +93,6 @@ select = ["E", "F", "I", "UP", "B"] "tools/review/chunker_review.py" = ["E501"] "tools/eval_dataset/review.py" = ["E501"] "tools/evaluation/preview.py" = ["E501"] -# news.py is a standalone script, not part of the package -"news.py" = ["E501", "B904"] [tool.ruff.format] quote-style = "double" From 1b0351ade6b21ac724128da53aea06b58c459e6f Mon Sep 17 00:00:00 2001 From: henryle97 Date: Fri, 7 Aug 2026 14:07:20 +0700 Subject: [PATCH 4/5] chore: satisfy the repo's own pre-commit hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem. `pre-commit run --all-files` had never been run to completion, so two hooks failed on code already in main. end-of-file-fixer flagged seven files — three survey docs missing a trailing newline, and CLAUDE.md plus two parser scripts carrying extra trailing blank lines. gitleaks reported three findings, all false positives, which meant the secret-scanning hook failed for every contributor who installed it. What changed. Applied the end-of-file fixes. Added .gitleaksignore with the three reviewed false positives: `dedup_family=True`, a retrieval config flag that matches the generic-api-key shape, and two `Authorization: Bearer rk_live_...` lines in an API design document whose trailing ellipsis is literal. Each entry records why the match is safe, and the file warns that fingerprints are line-sensitive and must be re-verified rather than blindly refreshed. `pre-commit run --all-files` now passes clean, so the hook suite is trustworthy before the repo goes public. What did NOT change. No behavior. gitleaks still reports the same two placeholder matches when scanning full history at their pre-rename path; they are the same reviewed false positives. Co-Authored-By: Claude Opus 5 (1M context) --- .gitleaksignore | 17 +++++++++++++++++ CLAUDE.md | 2 -- docs/survey/SOTA-RAG-for-financial-documents.md | 2 +- docs/survey/claude-rag-survey.md | 2 +- docs/survey/datasets.md | 2 +- docs/survey/grok-rag-survey.md | 4 ++-- scripts/parser/parse_f1_doc.sh | 1 - scripts/parser/parse_pj_doc.sh | 1 - 8 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 .gitleaksignore diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..8ef8dd8 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,17 @@ +# Reviewed gitleaks false positives. +# +# Fingerprints are :: and are line-sensitive: if one of +# these lines moves, gitleaks re-reports it and the fingerprint below needs +# updating. Re-verify that the match is still a placeholder before doing so — +# never add a fingerprint without reading the line it points at. + +# `dedup_family=True` is a retrieval config flag. The generic-api-key rule +# matches the `=` shape, not anything secret. This entry is a +# table of doc snippets asserted to stay in sync with the default config. +tests/unit/test_client_app_config.py:generic-api-key:94 + +# `Authorization: Bearer rk_live_...` in an API design document. The literal +# trailing ellipsis is part of the text — this is an illustrative curl example +# for a proposed per-tenant key format, not an issued credential. +docs/sub-plans/api/apis_design.md:curl-auth-header:685 +docs/sub-plans/api/apis_design.md:curl-auth-header:705 diff --git a/CLAUDE.md b/CLAUDE.md index 0221fe0..0cd76ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,5 +83,3 @@ Single-context repo — `CONTEXT.md` and `docs/adr/` at the repo root (created l ## Rules - gh use the `henryle97` account. Run `gh auth switch -u henryle97` if needed - - diff --git a/docs/survey/SOTA-RAG-for-financial-documents.md b/docs/survey/SOTA-RAG-for-financial-documents.md index e8b406f..bb3da8c 100644 --- a/docs/survey/SOTA-RAG-for-financial-documents.md +++ b/docs/survey/SOTA-RAG-for-financial-documents.md @@ -424,4 +424,4 @@ When you're ready for multi-company (Level 2), you download Microsoft and Tesla **You never index FinanceBench itself.** It's purely evaluation data — like how you never put your unit test assertions into your application database. -Does that clear it up? \ No newline at end of file +Does that clear it up? diff --git a/docs/survey/claude-rag-survey.md b/docs/survey/claude-rag-survey.md index 7c755f2..9a5bc98 100644 --- a/docs/survey/claude-rag-survey.md +++ b/docs/survey/claude-rag-survey.md @@ -559,4 +559,4 @@ Step 6.5 Private-set robustness measures: - **GaRAGe** (ACL 2025) — Grounding annotations benchmark for RAG evaluation - **RAGAS** — Automated RAG evaluation framework - **MinerU** (OpenDataLab) — 97.5 mAP layout detection, best table extraction -- **Docling** (IBM) — MIT-licensed, TableFormer-powered parsing \ No newline at end of file +- **Docling** (IBM) — MIT-licensed, TableFormer-powered parsing diff --git a/docs/survey/datasets.md b/docs/survey/datasets.md index 41a6384..fc7046c 100644 --- a/docs/survey/datasets.md +++ b/docs/survey/datasets.md @@ -1,6 +1,6 @@ # Datasets -- **FinanceBench** +- **FinanceBench** - **T2-RAGBench** - FinDER Here's the landscape of RAG evaluation datasets, organized from most to least relevant for your financial report use case: diff --git a/docs/survey/grok-rag-survey.md b/docs/survey/grok-rag-survey.md index e2471b8..6e735b0 100644 --- a/docs/survey/grok-rag-survey.md +++ b/docs/survey/grok-rag-survey.md @@ -61,6 +61,6 @@ For a deployable system today (English SEC/earnings reports): - Add post-retrieval re-ranking and citation/attribution (cell-level in HierFinRAG). - For scale: Binary quantization + hybrid search reduces costs dramatically. -These approaches (especially HierFinRAG + Unstructured.io backbone) represent the current English-language SOTA for reliable, hallucination-resistant RAG on text + table financial reports. They move far beyond vanilla RAG by explicitly modeling the interplay between narrative and numerical data. +These approaches (especially HierFinRAG + Unstructured.io backbone) represent the current English-language SOTA for reliable, hallucination-resistant RAG on text + table financial reports. They move far beyond vanilla RAG by explicitly modeling the interplay between narrative and numerical data. -If you need code snippets, a specific implementation guide (e.g., LlamaIndex + Unstructured + HierFinRAG components), or evaluation on a sample 10-K, let me know! \ No newline at end of file +If you need code snippets, a specific implementation guide (e.g., LlamaIndex + Unstructured + HierFinRAG components), or evaluation on a sample 10-K, let me know! diff --git a/scripts/parser/parse_f1_doc.sh b/scripts/parser/parse_f1_doc.sh index 565db4c..2ec01f9 100644 --- a/scripts/parser/parse_f1_doc.sh +++ b/scripts/parser/parse_f1_doc.sh @@ -128,4 +128,3 @@ PY # uv run python -m xrag.cli parser \ # --config data/dev/ir_document/configs/parsers/parse-f1-doc.yml - diff --git a/scripts/parser/parse_pj_doc.sh b/scripts/parser/parse_pj_doc.sh index d124ffd..f6e50e9 100644 --- a/scripts/parser/parse_pj_doc.sh +++ b/scripts/parser/parse_pj_doc.sh @@ -126,4 +126,3 @@ PY # uv run python -m xrag.cli parser \ # --config data/dev/ir_document/configs/parsers/parse-f1-doc.yml - From b9814df2060fd1af81e59e6777f50abfd5ab65b5 Mon Sep 17 00:00:00 2001 From: henryle97 Date: Fri, 7 Aug 2026 14:08:05 +0700 Subject: [PATCH 5/5] fix(precommit): bump gitleaks to a rev that loads on current macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinned v8.21.2 builds a binary that recent macOS dyld refuses to load, aborting with "missing LC_UUID load command". pre-commit surfaces that as exit -6, which reads like a hook failure rather than the crash it is — so the secret scanner appeared to be running and failing when it was never running at all. Bumped to v8.30.1, which loads and passes, and left a comment recording the failure mode so the next person to see exit -6 knows to bump rather than to hunt for a leak. Co-Authored-By: Claude Opus 5 (1M context) --- .pre-commit-config.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ecfb74..4540952 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,9 @@ repos: args: [--branch, main] - repo: https://github.com/gitleaks/gitleaks - rev: v8.21.2 + # Keep this reasonably current. Binaries built from older revs (v8.21.2 + # and earlier) fail to load on recent macOS with a dyld "missing LC_UUID" + # abort, which surfaces as a hook crash (exit -6) rather than a finding. + rev: v8.30.1 hooks: - id: gitleaks