diff --git a/.codex b/.codex deleted file mode 100644 index e69de29..0000000 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..89ae507 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,84 @@ +name: Bug report +description: Something in xrag behaves incorrectly +labels: ["needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for filing a bug. Please redact all API keys and any + confidential document content before submitting. + + For security vulnerabilities, follow + [SECURITY.md](https://github.com/henryle97/xrag/blob/main/SECURITY.md) + instead of opening a public issue. + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What did you observe, and what did you expect instead? + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: > + A minimal snippet or CLI invocation. Include the relevant config with + credentials redacted. + render: shell + validations: + required: true + + - type: input + id: version + attributes: + label: xrag version + description: "Output of `python -c \"import xrag; print(xrag.__version__)\"`" + placeholder: "0.5.3" + validations: + required: true + + - type: input + id: python-version + attributes: + label: Python version + placeholder: "3.12.9" + validations: + required: true + + - type: dropdown + id: area + attributes: + label: Which area? + options: + - parser + - chunker + - enrichment + - embedding + - indexing / vector store + - retrieval / rerank + - generation + - Xrag client + - CLI + - eval / benchmark + - packaging / install + - not sure + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Logs or traceback + description: Paste the full traceback if there is one. Redact credentials. + render: shell + + - type: textarea + id: context + attributes: + label: Anything else? + description: > + Vector store backend, provider models, document type, or anything else + that helps reproduce. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..c91fd72 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Question or usage help + url: https://github.com/henryle97/xrag/discussions + about: Ask how to use xrag, or discuss an idea before filing a feature request. + - name: Security vulnerability + url: https://github.com/henryle97/xrag/security/advisories/new + about: Report privately. Please do not open a public issue for security problems. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..dfbddc5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,54 @@ +name: Feature request +description: Suggest a capability or improvement for xrag +labels: ["needs-triage"] +body: + - type: textarea + id: problem + attributes: + label: What problem are you trying to solve? + description: > + Describe the use case rather than a specific implementation. What are + you unable to do today? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: What would the API, config, or CLI surface look like? + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Workarounds you tried, or other designs you weighed. + + - type: dropdown + id: area + attributes: + label: Which area? + options: + - parser + - chunker + - enrichment + - embedding + - indexing / vector store + - retrieval / rerank + - generation + - Xrag client + - CLI + - eval / benchmark + - packaging / install + - not sure + validations: + required: true + + - type: checkboxes + id: contribution + attributes: + label: Contribution + options: + - label: I'd be willing to open a PR for this diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e9bc56f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,36 @@ +version: 2 +updates: + # Keep CI actions current — these are the highest-value updates for a + # library, since a compromised action runs with repo credentials. + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + labels: ["needs-triage"] + commit-message: + prefix: "ci" + + # Python dependencies. Grouped so routine patch bumps arrive as one PR + # rather than flooding the queue. + - package-ecosystem: uv + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + labels: ["needs-triage"] + commit-message: + prefix: "chore(deps)" + groups: + patch-and-minor: + update-types: + - minor + - patch + ignore: + # LangChain majors are pinned deliberately and exercised by the contract + # matrix in ci.yml. Bump these by hand alongside that matrix. + - dependency-name: "langchain" + update-types: ["version-update:semver-major"] + - dependency-name: "langchain-core" + update-types: ["version-update:semver-major"] + - dependency-name: "langchain-openai" + update-types: ["version-update:semver-major"] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ce1fed4..9faf2a8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,5 @@ @@ -24,7 +24,7 @@ ONE `Closes #N` per issue, each on its own content line. GitHub's auto-close parser only fires on `Closes`/`Fixes`/`Resolves` when the keyword is on the same content line as the reference — a heading like `## Closes` followed by `#19, #20, #21` will silently -fail to close anything. See docs/contributing.md §3.1. +fail to close anything. See CONTRIBUTING.md §3.1. Single-issue example: Closes #42 diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..8ef8dd8 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,17 @@ +# Reviewed gitleaks false positives. +# +# Fingerprints are :: and are line-sensitive: if one of +# these lines moves, gitleaks re-reports it and the fingerprint below needs +# updating. Re-verify that the match is still a placeholder before doing so — +# never add a fingerprint without reading the line it points at. + +# `dedup_family=True` is a retrieval config flag. The generic-api-key rule +# matches the `=` shape, not anything secret. This entry is a +# table of doc snippets asserted to stay in sync with the default config. +tests/unit/test_client_app_config.py:generic-api-key:94 + +# `Authorization: Bearer rk_live_...` in an API design document. The literal +# trailing ellipsis is part of the text — this is an illustrative curl example +# for a proposed per-tenant key format, not an issued credential. +docs/sub-plans/api/apis_design.md:curl-auth-header:685 +docs/sub-plans/api/apis_design.md:curl-auth-header:705 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ecfb74..4540952 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,9 @@ repos: args: [--branch, main] - repo: https://github.com/gitleaks/gitleaks - rev: v8.21.2 + # Keep this reasonably current. Binaries built from older revs (v8.21.2 + # and earlier) fail to load on recent macOS with a dyld "missing LC_UUID" + # abort, which surfaces as a hook crash (exit -6) rather than a finding. + rev: v8.30.1 hooks: - id: gitleaks diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..02a141d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,106 @@ +# Changelog + +All notable changes to this project are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Only names exported from `xrag.__all__` are covered by the version contract. +Anything outside that surface is internal and may change in any release. + +## [Unreleased] + +### Added + +- DOCX image extraction: `DocxPicturePartitioner` and an `extract_images` + parser option (#61) +- Image chunks in `SectionTableChunker` (#62) +- LLM-generated image descriptions (#64) + +### Changed + +- Aligned retrieval and indexing defaults (#67) + +## [0.5.3] — 2026-05-05 + +### Added + +- Navigational testset generator, with ADR-0003 recording the primary-chunk + evidence decision (#51) +- Navigation questions in dataset generation (#54) +- Per-category metric breakdown by `reasoning_type` (#52) +- Append mode for the `eval create-dataset` CLI (#53) + +### Changed + +- Indexing now augments retrieval text instead of replacing it, per ADR-0002 + (#45) + +## [0.5.2] — 2026-05-05 + +### Fixed + +- Pin `openai_api_type=None` in the embedder to stop an environment variable + leaking into requests (#42) + +## [0.5.1] — 2026-05-05 + +### Added + +- `ChunkMetadata` promoted to the public `__all__` surface (#40) + +## [0.5.0] — 2026-05-04 + +### Added + +- Per-step `max_concurrency` for LLM-bound enrichers (#39) + +### Changed + +- **Breaking:** unified the chunk schema on the canonical V2 envelope and + `doc_id` (#37) + +## [0.4.1] — 2026-05-04 + +### Added + +- `qdrant_api_key` parameter on `Xrag.__init__` (#29) + +## [0.4.0] — 2026-05-03 + +### Changed + +- Explicit settings injection across the asymmetric boundary, recorded in + ADR-0001 (#26) + +## [0.3.0] — 2026-05-01 + +### Changed + +- Library and client SDK cleanup pass across the public surface (#3) + +## [0.2.0] — 2026-04-27 + +### Added + +- High-level async `Xrag` client with resource-namespaced operations + (`documents.*`, `retrievals.*`, `rag.*`) +- Tenant binding via `client.for_tenant(...)` +- Typed error hierarchy rooted at `XragError` +- Split the local-PDF parser into its own `parser-unstructured-local-pdf` extra + +### Changed + +- Packaging moved to the hatch backend; the project publishes to PyPI as + `pyxrag` while the import name stays `xrag` +- Renamed the engine repository from `fin-rag` to `xrag` + +[Unreleased]: https://github.com/henryle97/xrag/compare/v0.5.3...HEAD +[0.5.3]: https://github.com/henryle97/xrag/compare/v0.5.2...v0.5.3 +[0.5.2]: https://github.com/henryle97/xrag/compare/v0.5.1...v0.5.2 +[0.5.1]: https://github.com/henryle97/xrag/compare/v0.5.0...v0.5.1 +[0.5.0]: https://github.com/henryle97/xrag/compare/v0.4.1...v0.5.0 +[0.4.1]: https://github.com/henryle97/xrag/compare/v0.4.0...v0.4.1 +[0.4.0]: https://github.com/henryle97/xrag/compare/v0.3.0...v0.4.0 +[0.3.0]: https://github.com/henryle97/xrag/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/henryle97/xrag/releases/tag/v0.2.0 diff --git a/CLAUDE.md b/CLAUDE.md index 0221fe0..0cd76ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,5 +83,3 @@ Single-context repo — `CONTEXT.md` and `docs/adr/` at the repo root (created l ## Rules - gh use the `henryle97` account. Run `gh auth switch -u henryle97` if needed - - diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..6bb9929 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,126 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples include using an official email address, posting via an official social +media account, or acting as an appointed representative at an online or offline +event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the project maintainers by opening a +[private security advisory](https://github.com/henryle97/xrag/security/advisories/new), +which is visible only to maintainers, or by contacting +[@henryle97](https://github.com/henryle97) directly on GitHub. + +All complaints will be reviewed and investigated promptly and fairly. All +community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. Violating +these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. Violating these +terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla]. + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. + +[homepage]: https://www.contributor-covenant.org +[mozilla]: https://github.com/mozilla/diversity diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..484dcbb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,150 @@ +# Contributing to xrag + +Thanks for your interest in xrag. This document covers how to get set up, the +checks we expect before review, and the conventions the PR template encodes. + +## 1. Getting set up + +xrag uses [uv](https://docs.astral.sh/uv/) for dependency management and +requires Python 3.11 or newer. + +```bash +git clone https://github.com/henryle97/xrag.git +cd xrag +make sync # uv sync --group dev + the extras the tests need +make precommit-install # installs the pre-commit hooks (recommended) +``` + +`make sync` installs the `cli`, `chroma`, and `rerank-cohere` extras because +the unit tests import them directly. A bare `uv sync` is not enough to run the +test suite. + +Local PDF/OCR parsing is heavy and optional. Add it only if you're working on +that path: + +```bash +uv sync --group dev --extra cli --extra chroma --extra rerank-cohere \ + --extra parser-unstructured-local-pdf +``` + +### Environment variables + +Copy `.env.example` to `.env` and fill in only the keys you need. Every +variable maps to a field on `xrag.config.settings.Settings`. Unit tests are +hermetic and need no credentials. + +## 2. Development workflow + +Run these before opening a PR: + +```bash +make lint # ruff check + ruff format --check +make unit-test # pytest tests/unit -q — hermetic, this is the CI gate +``` + +`make check` additionally runs `compileall`. Integration tests +(`make integration-test`) make real network calls, are not part of CI, and +require credentials. + +After changing runtime behavior, also exercise the changed path with a live CLI +command — unit tests alone don't prove a pipeline still runs end to end. + +### Tests + +- Unit tests live in `tests/unit/` and must stay hermetic — no network, no + credentials, no reliance on files outside the repo. +- Contract tests in `tests/contract/` run against the built wheel and guard the + public API surface. If you add or remove anything in `xrag.__all__`, update + them. +- Mark network-dependent tests with `requires_network`, `requires_qdrant`, or + `requires_openai`. + +### Public API surface + +Everything exported from `xrag.__all__` is the supported surface and is +SemVer-bound. Anything outside it is internal. Changing an exported name is a +breaking change and needs a contract-test update plus a changelog entry. + +## 3. Pull requests + +The [PR template](.github/pull_request_template.md) asks for four things. +Keep the shape: + +- **Problem** — what's broken or missing +- **Why** — why it's worth changing now +- **What changed** — the actual diff, in prose +- **What did NOT change** — scope boundaries, so reviewers know where not to look + +Include a **Test plan** with `make lint` and `make unit-test` checked, plus any +live CLI, integration, or manual verification you did. Skip the Risks section +entirely when there are none rather than writing "None". + +Other expectations: + +- Branch from an up-to-date `main`: `git checkout main && git pull --ff-only origin main` +- Keep PRs focused. Parent/child issues ship together as the parent's PR. +- Resolve review threads only after the fix is pushed. +- Don't auto-merge. + +### 3.1 Closing issues + +GitHub's auto-close parser only fires when a close keyword (`Closes`, `Fixes`, +`Resolves`) sits on the same content line as the issue reference. A heading like +`## Closes` followed by `#19, #20, #21` **silently closes nothing**. + +Write one close keyword per issue, each on its own content line: + +```text +Closes #19. Closes #20. Closes #21. +``` + +Verify before merging: + +```bash +gh pr view --json closingIssuesReferences +``` + +## 4. Commit conventions + +We use [Conventional Commits](https://www.conventionalcommits.org/) with a +module scope: + +```text +feat(parser): add DocxPicturePartitioner +fix(config): reject unknown provider names +test(chunker): cover image chunk emission +docs(plans): refresh indexing backlog +``` + +Common scopes: `parser`, `chunker`, `embedder`, `indexer`, `client`, `config`, +`eval`, `ci`, `packaging`. + +## 5. Architecture decisions + +Check [`docs/adr/`](docs/adr/) before making architectural changes — it records +decisions that are easy to accidentally reverse. Add a new ADR when you change +a cross-module contract or reverse an existing decision. + +Deeper domain context lives in [`CONTEXT.md`](CONTEXT.md) and +[`docs/agents/domain.md`](docs/agents/domain.md). + +## 6. Evaluation and benchmarks + +Retrieval quality changes need evidence. The full discipline is documented in +[`CLAUDE.md`](CLAUDE.md); the short version: + +- Run evals through `scripts/eval/`, which writes to `experiments/eval/-/`. +- **Append** a new row to [`docs/BENCHMARK.md`](docs/BENCHMARK.md) — never edit + existing rows. +- Per-category `R@10` is the primary regression signal. +- `experiments/eval/experiments.jsonl` is the append-only source of truth. + +Benchmark documentation uses a fictional issuer and synthetic figures. Never +commit real customer documents, real financial data, or anything under `data/` +(it is gitignored for this reason). + +## 7. Reporting bugs and requesting features + +Use the [issue templates](https://github.com/henryle97/xrag/issues/new/choose). +For security issues, follow [SECURITY.md](SECURITY.md) instead of opening a +public issue. diff --git a/Makefile b/Makefile index 404bc19..bca996c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .PHONY: sync lint format check unit-test integration-test parser parser-smoke precommit-install precommit-run +# Extras the unit tests import directly (chromadb, cohere, typer). Kept in sync +# with the `unit-test` job in .github/workflows/ci.yml. +DEV_EXTRAS := --extra cli --extra chroma --extra rerank-cohere + sync: - uv sync + uv sync --group dev $(DEV_EXTRAS) lint: uv run ruff check . @@ -14,7 +18,7 @@ check: lint python3 -m compileall xrag examples tools unit-test: - uv run pytest tests/unit -q + uv run $(DEV_EXTRAS) pytest tests/unit -q integration-test: uv run pytest tests/integration -q diff --git a/README.md b/README.md index ee6433f..e7116fa 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ Recommended path: 7. Use [`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md) for the high-level Python client design. 8. Use [`docs/plans.md`](docs/plans.md) and [`docs/sub-plans/`](docs/sub-plans/) for roadmap and implementation details. -## Quickstart — library (v0.1, functional API) +## Quickstart — library (functional API) ```python from pathlib import Path @@ -369,7 +369,11 @@ result = run_rag("What was Cash as of December 31, 2024?", cfg) print(result.answer) ``` -The high-level `Xrag` async client (`client.documents.ingest(...)`, `client.retrievals.search(...)`, `client.rag.ask(...)`) ships in v0.2 — see [`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md). +Most applications should prefer the high-level `Xrag` client shown above. Use +the functional API when you already work in terms of `AppConfig` and want +direct control over pipeline wiring — see +[`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md) for the +design behind both surfaces. ## Quickstart — CLI @@ -414,7 +418,7 @@ uv run python -m xrag.cli parser preprocess \ # Chunk uv run python -m xrag.cli chunk prepare \ --input data/dev/ir_document/normalized/elements.json \ - --config configs/chunker.yml \ + --config configs/chunkers/section-table.yml \ --output-dir data/dev/ir_document/chunked # Baseline RAG @@ -447,15 +451,25 @@ UNSTRUCTURED_API_URL=... # optional For DOCX-to-PDF conversion install LibreOffice so `soffice` / `libreoffice` is on `PATH`. -## Phasing +## Project status -| Version | Surface | -|---|---| -| **v0.1** (current) | Functional API: `run_ingest`, `run_rag`, Pydantic models, configs, loaders. | -| **v0.2** | High-level `Xrag` async client with resource-namespaced ops (`documents.*`, `retrievals.*`, `rag.*`). Tenant binding via `for_tenant`. Typed error hierarchy. | -| **v0.3** | `XragSync` mirror. | -| **v0.4** | Per-call `config_overrides`, batch ingest, persistent registry contract. | -| **v0.5+ (backlog)** | Streaming surfaces (`documents.ingest_stream`, `rag.ask_stream`). | +`xrag` is pre-1.0. Everything exported from `xrag.__all__` is the supported +surface; anything outside it is internal and not SemVer-bound. Minor releases +may still change internals. + +**Available today:** + +- Functional API — `run_ingest`, `run_rag`, Pydantic models, config loaders +- High-level async `Xrag` client with resource-namespaced ops (`documents.*`, + `retrievals.*`, `rag.*`) +- Tenant binding via `client.for_tenant(...)` +- Typed error hierarchy (`XragError` and subclasses) +- Per-call ingest and retrieval overrides + +**Planned:** + +- `XragSync` — a synchronous mirror of the async client +- Streaming surfaces (`documents.ingest_stream`, `rag.ask_stream`) Full design: [`docs/sub-plans/xrag-public-api.md`](docs/sub-plans/xrag-public-api.md). @@ -492,8 +506,8 @@ uv sync --group dev --extra cli --extra chroma --extra rerank-cohere --extra par - Parser provider: `unstructured`. - Preprocess config: [`configs/preprocess.yml`](configs/preprocess.yml). -- Default chunker: `section_table` in [`configs/chunker.yml`](configs/chunker.yml). -- Alternative chunker: `ragflow` in [`configs/chunker-ragflow.yml`](configs/chunker-ragflow.yml). +- Default chunker: `section_table` in [`configs/chunkers/section-table.yml`](configs/chunkers/section-table.yml). +- Alternative chunkers: `section_token` (alias `ragflow`) in [`configs/chunkers/section-token.yml`](configs/chunkers/section-token.yml), and `title_hierarchy` in [`configs/chunkers/title-hierarchy.yml`](configs/chunkers/title-hierarchy.yml). - Simple one-shot RAG config: [`configs/rag-simple-baseline.yml`](configs/rag-simple-baseline.yml). - Persistent index/query config: [`configs/rag-baseline.yml`](configs/rag-baseline.yml). - Enrichment scenario configs: [`configs/enrichment/`](configs/enrichment). @@ -507,7 +521,8 @@ uv sync --group dev --extra cli --extra chroma --extra rerank-cohere --extra par - [`tools/`](tools/): **dev-only** — eval scoring, dataset generation, HTML viewers, eval datasets. Excluded from the wheel. - [`configs/`](configs/): runtime YAML configs. - [`docs/`](docs/README.md): user docs, plans, surveys, and terminology. -- [`data/`](data/): local inputs and generated artifacts. +- `data/`: local inputs and generated artifacts. Gitignored — created on first + run, never committed. ## Development checks @@ -530,6 +545,17 @@ GitHub Actions now verifies the same core paths contributors should run locally: Tag pushes matching `v*` reuse the verified package artifacts and publish them to the GitHub Release instead of rebuilding a second time during release. +## Contributing + +Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, +the checks expected before review, and PR conventions. Please also read the +[Code of Conduct](CODE_OF_CONDUCT.md). + +Release history is in [CHANGELOG.md](CHANGELOG.md). + +To report a security issue, follow [SECURITY.md](SECURITY.md) rather than +opening a public issue. + ## License MIT — see [LICENSE](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2fc217f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,52 @@ +# Security Policy + +## Supported versions + +xrag is pre-1.0. Security fixes land on the latest minor release only. Please +upgrade before reporting an issue against an older version. + +| Version | Supported | +| ------- | --------- | +| 0.5.x | ✅ | +| < 0.5 | ❌ | + +## Reporting a vulnerability + +**Please do not open a public issue for security problems.** + +Report privately through GitHub's +[private vulnerability reporting](https://github.com/henryle97/xrag/security/advisories/new) +for this repository. If that is unavailable to you, open a public issue +containing only a request for a private contact channel — no details. + +Please include: + +- A description of the issue and its impact +- Steps to reproduce, or a proof-of-concept +- Affected version and relevant configuration (redact all credentials) + +You can expect an initial response within 7 days and a status update at least +every 14 days until the issue is resolved. + +## Scope + +xrag is a library that talks to third-party services (OpenAI, Cohere, Qdrant, +Unstructured, AWS Bedrock). Vulnerabilities in those services belong to their +respective vendors. In scope for this project: + +- Credential handling and leakage through logs, artifacts, traces, or errors +- Tenant isolation failures in `client.for_tenant(...)` or collection scoping +- Path traversal or unsafe file handling in the parser and artifact-store paths +- Deserialization or injection issues in config, chunk, or dataset loading +- Dependency vulnerabilities reachable through xrag's default code paths + +## Handling credentials + +xrag reads credentials from environment variables only — never from config +YAML. When filing any issue or sharing a config, redact `OPENAI_API_KEY`, +`COHERE_API_KEY`, `QDRANT_API_KEY`, `UNSTRUCTURED_API_KEY`, AWS credentials, +and tracing keys. + +This repository runs [gitleaks](https://github.com/gitleaks/gitleaks) as a +pre-commit hook. Install the hooks with `make precommit-install` so secrets are +caught before they reach a commit. diff --git a/configs/baseline.yml b/configs/baseline.yml index 52973eb..b361652 100644 --- a/configs/baseline.yml +++ b/configs/baseline.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/all.yml b/configs/enrichment/all.yml index db849b9..fbc9566 100644 --- a/configs/enrichment/all.yml +++ b/configs/enrichment/all.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/auto-keywords.yml b/configs/enrichment/auto-keywords.yml index f3d4a37..7b8186f 100644 --- a/configs/enrichment/auto-keywords.yml +++ b/configs/enrichment/auto-keywords.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/auto-questions.yml b/configs/enrichment/auto-questions.yml index 85f46c8..de68b0e 100644 --- a/configs/enrichment/auto-questions.yml +++ b/configs/enrichment/auto-questions.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/table-context.yml b/configs/enrichment/table-context.yml index f65e99f..774d3d9 100644 --- a/configs/enrichment/table-context.yml +++ b/configs/enrichment/table-context.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/enrichment/table-summary.yml b/configs/enrichment/table-summary.yml index 68bcb90..3a73dd2 100644 --- a/configs/enrichment/table-summary.yml +++ b/configs/enrichment/table-summary.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/raw/ir_document/Pj_AVATAR_F_page_260205.docx" + - path: "data/raw/ir_document/ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/parsers/parse-f1-doc-local.yml b/configs/parsers/parse-f1-doc-local.yml index a866082..4435f25 100644 --- a/configs/parsers/parse-f1-doc-local.yml +++ b/configs/parsers/parse-f1-doc-local.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # override with --output-dir sources: - - path: "data/raw/ir_document/F1-file.docx" + - path: "data/raw/ir_document/ir-sample-b.docx" doc_id: f1-doc parser: diff --git a/configs/parsers/parse-f1-doc.yml b/configs/parsers/parse-f1-doc.yml index a2b2120..f11f108 100644 --- a/configs/parsers/parse-f1-doc.yml +++ b/configs/parsers/parse-f1-doc.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # defauloverwrite by --output-dir sources: - - path: "data/raw/ir_document/F1-file.docx" + - path: "data/raw/ir_document/ir-sample-b.docx" doc_id: f1-doc parser: diff --git a/configs/parsers/parse-pj-doc-local.yml b/configs/parsers/parse-pj-doc-local.yml index 59b696a..e2ec9ff 100644 --- a/configs/parsers/parse-pj-doc-local.yml +++ b/configs/parsers/parse-pj-doc-local.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # override with --output-dir sources: - - path: "data/raw/ir_document/Pj_AVATAR_F_page_260205.docx" + - path: "data/raw/ir_document/ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/parsers/parse-pj-doc.yml b/configs/parsers/parse-pj-doc.yml index 5545da4..5a121c3 100644 --- a/configs/parsers/parse-pj-doc.yml +++ b/configs/parsers/parse-pj-doc.yml @@ -4,7 +4,7 @@ pipeline: # artifact_dir: data/dev/ir_document/parser_outputs # defauloverwrite by --output-dir sources: - - path: "Pj_AVATAR_F_page_260205.docx" + - path: "ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/rag-avatar-eval.yml b/configs/rag-avatar-eval.yml index 1e9770c..0aa2299 100644 --- a/configs/rag-avatar-eval.yml +++ b/configs/rag-avatar-eval.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-baseline-ragflow.yml b/configs/rag-ir-baseline-ragflow.yml index 7378c0d..a75eaf2 100644 --- a/configs/rag-ir-baseline-ragflow.yml +++ b/configs/rag-ir-baseline-ragflow.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-baseline.yml b/configs/rag-ir-baseline.yml index 7b5e19b..e53eb10 100644 --- a/configs/rag-ir-baseline.yml +++ b/configs/rag-ir-baseline.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-hybrid.yml b/configs/rag-ir-hybrid.yml index e5a286e..bcd7d78 100644 --- a/configs/rag-ir-hybrid.yml +++ b/configs/rag-ir-hybrid.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-latest.yml b/configs/rag-ir-latest.yml index e42287c..c620c89 100644 --- a/configs/rag-ir-latest.yml +++ b/configs/rag-ir-latest.yml @@ -1,15 +1,15 @@ pipeline: name: rag-ir-all-docs-latest mode: full - artifact_dir: /data/lucas/projects/rag/ir_v2/data/dev/ir_document + artifact_dir: data/dev/ir_document tenant_id: dev doc_ids: - f1-doc - pj-doc sources: -- path: /data/lucas/projects/rag/ir_v2/data/dev/ir_document/raw/F1-file.docx +- path: data/dev/ir_document/raw/ir-sample-b.docx doc_id: f1-doc -- path: /data/lucas/projects/rag/ir_v2/data/dev/ir_document/raw/Pj_AVATAR_F_page_260205.docx +- path: data/dev/ir_document/raw/ir-sample-a.docx doc_id: pj-doc parser: provider: unstructured_local @@ -53,7 +53,7 @@ vector_store: provider: qdrant options: collection_name: ir_document_latest - path: /data/lucas/projects/rag/ir_v2/data/dev/ir_document/qdrant_db + path: data/dev/ir_document/qdrant_db distance: cosine indexing: exclude_parents_from_index: true diff --git a/configs/rag-ir-mvp1.yml b/configs/rag-ir-mvp1.yml index 4a8b637..b5a2d73 100644 --- a/configs/rag-ir-mvp1.yml +++ b/configs/rag-ir-mvp1.yml @@ -22,7 +22,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-mvp2-rerank.yml b/configs/rag-ir-mvp2-rerank.yml index 6b8d2d5..4851353 100644 --- a/configs/rag-ir-mvp2-rerank.yml +++ b/configs/rag-ir-mvp2-rerank.yml @@ -16,7 +16,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-mvp2.yml b/configs/rag-ir-mvp2.yml index f0d9325..db07247 100644 --- a/configs/rag-ir-mvp2.yml +++ b/configs/rag-ir-mvp2.yml @@ -14,7 +14,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-mvp3-all-docs.yml b/configs/rag-ir-mvp3-all-docs.yml index 637c6b8..23c46a2 100644 --- a/configs/rag-ir-mvp3-all-docs.yml +++ b/configs/rag-ir-mvp3-all-docs.yml @@ -8,9 +8,9 @@ pipeline: doc_ids: [f1-doc, pj-doc] sources: - - path: "data/raw/ir_document/F1-file.docx" + - path: "data/raw/ir_document/ir-sample-b.docx" doc_id: f1-doc - - path: "data/raw/ir_document/Pj_AVATAR_F_page_260205.docx" + - path: "data/raw/ir_document/ir-sample-a.docx" doc_id: pj-doc parser: diff --git a/configs/rag-ir-mvp3.yml b/configs/rag-ir-mvp3.yml index 60cdc13..56681c2 100644 --- a/configs/rag-ir-mvp3.yml +++ b/configs/rag-ir-mvp3.yml @@ -17,7 +17,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/artifacts/ir_document/raw/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/artifacts/ir_document/raw/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir-rerank.yml b/configs/rag-ir-rerank.yml index 4eadf4b..bb8395f 100644 --- a/configs/rag-ir-rerank.yml +++ b/configs/rag-ir-rerank.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-ir.yml b/configs/rag-ir.yml index 6c900c6..0a2034e 100644 --- a/configs/rag-ir.yml +++ b/configs/rag-ir.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/configs/rag-simple-baseline.yml b/configs/rag-simple-baseline.yml index b20f10d..4512371 100644 --- a/configs/rag-simple-baseline.yml +++ b/configs/rag-simple-baseline.yml @@ -4,7 +4,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/docs/BENCHMARK-chunk.md b/docs/BENCHMARK-chunk.md index f842e63..d36e389 100644 --- a/docs/BENCHMARK-chunk.md +++ b/docs/BENCHMARK-chunk.md @@ -113,7 +113,7 @@ All on gpt-4.1-nano for enrichment + generation, `text-embedding-3-small` for em 1. **`table_summary` did NOT help on this testset.** Clean A/B at the same retrieval stack (rows 04b vs 05) shows `num_match` *regressed* by −0.037 with the summary on, MRR was effectively flat (−0.003), substring tied. Earlier "row 04 vs row 01" delta was *real* but attributable to the hybrid retrieval stack (BM25 + RRF + dedup_family), not the summary. 2. **Cohere reranker did NOT help on this testset.** Rows 04 vs 04b show `num_match` flat (0.926 = 0.926), MRR +0.002, substring *worse* −0.026 with rerank on. Rerank costs +3651ms p95 retrieval latency for negative quality movement on a 39-q testset. 3. **The hybrid retrieval stack alone (BM25 + vector + RRF + dedup_family)** is the lever responsible for the gains over the simple-retrieval baseline. Row 05 (no summary, no rerank, just hybrid) lifts num_match from 0.852 to 0.963 vs row 01 — a +0.111 gain attributable purely to the retriever change. Substring also +0.077, MRR +0.051. With ~no latency cost vs simple (450ms vs 437ms p95). -4. **Likely cause for the summary regression**: hallucinated numbers in summaries (observed in the inspection — chunk:56 "hundreds of millions" vs actual $8K–$438K; chunk:129 "$672M" vs actual ~$672K) shift the embedding into the wrong neighborhood. The LLM at prompt time still reads correct HTML, but it gets the *wrong table* fed in because retrieval ranked elsewhere. `auto_questions` (HyQE) is doing better natural-language anchoring without the numeric hallucination risk because the questions are query-shaped and don't have to commit to specific magnitudes. +4. **Likely cause for the summary regression**: hallucinated numbers in summaries (observed in the inspection — chunk:56 "hundreds of millions" vs actual $8K–$471K; chunk:129 "$540M" vs actual ~$540K) shift the embedding into the wrong neighborhood. The LLM at prompt time still reads correct HTML, but it gets the *wrong table* fed in because retrieval ranked elsewhere. `auto_questions` (HyQE) is doing better natural-language anchoring without the numeric hallucination risk because the questions are query-shaped and don't have to commit to specific magnitudes. 5. **`top_k_redundancy` improvements** (0.431 → 0.285–0.295 across rows 04/04b/05) are entirely from `dedup_family: true`, not from any enrichment. ### Practical takeaways @@ -161,7 +161,7 @@ All on gpt-4.1-nano for enrichment + generation, `text-embedding-3-small` for em string fragment across multiple chunks. 6. **`exact_match=0.000` everywhere.** The generator's normalized answer never matches the gold string verbatim. Expected — `gpt-4.1-nano` - paraphrases ("$7,351,900" vs "$7.35M"). Use `numeric_match` / + paraphrases ("$6,120,400" vs "$6.12M"). Use `numeric_match` / `substring_match` as the answer-grounding signal. ## Caveats / methodology notes diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index 1bb5c16..ba04157 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -49,7 +49,7 @@ testset: from it. **A correct refusal IS the right answer**, so a low score here is a metric artifact, not a generation defect. 2. **Terse factual answers**: byte-exact replies like `"17%"`, - `"January 2025"`, `"$4,431,280"` can be the answer to many unrelated + `"January 2025"`, `"$3,845,200"` can be the answer to many unrelated questions, so the synthetic-question step finds many low-similarity candidates and drives the mean down. Several **exact-match** answers in the multi-doc 87-Q set (1658 run) score 0.13–0.20. @@ -165,7 +165,7 @@ count would bring it under 1000ms. --- -## IR document (AVATAR / Neksai) — 40 LLM-generated questions +## IR document (AVATAR / Contoso) — 40 LLM-generated questions Dataset: `data/artifacts/ir_document/eval-llm/testset.jsonl` (40 Qs, 39 valid after validation). Categories: table_lookup (8), direct_lookup (15), semantic_paraphrase (8), multi_hop (6), no_answer (3). @@ -221,7 +221,7 @@ Embedding: `text-embedding-3-small` · Judge: `gpt-4.1-mini` · RAGAS metrics: ` | 01 | rag-ir-mvp1 | section_table | hybrid | 10 | gpt-4.1-mini | 0.973 | **0.96** | **0.83** | — | 543ms | **6267ms** | 622s | [100q-20260424-1126](../experiments/eval/rag-ir-mvp1-100q-20260424-1126/) | | 02 | rag-ir-mvp2 | section_table | hybrid | 10 | gpt-4.1-mini | 0.973 | **0.96** | 0.82 | — | **501ms** | 62791ms§ | 641s | [100q-20260424-1137](../experiments/eval/rag-ir-mvp2-100q-20260424-1137/) | -§ MVP2 `e2e p95=62791ms` is inflated by 5 queries with 60–91s generation times. Spot-check shows correct short answers (e.g. "$22,265,412" in 105 chars) — these are transient `gpt-4.1-mini` API stalls, not prompt-size or retrieval issues. MVP1 (same generator, ran 10 min earlier) saw max gen=9746ms. **Re-run recommended** to confirm MVP2 is not systematically slower. +§ MVP2 `e2e p95=62791ms` is inflated by 5 queries with 60–91s generation times. Spot-check shows correct short answers (e.g. "$24,830,170" in 105 chars) — these are transient `gpt-4.1-mini` API stalls, not prompt-size or retrieval issues. MVP1 (same generator, ran 10 min earlier) saw max gen=9746ms. **Re-run recommended** to confirm MVP2 is not systematically slower. ### MVP1 vs MVP2 on 100 Qs @@ -241,7 +241,7 @@ Quality is a statistical tie. The RAGFlow-style indexing changes in MVP2 (parent ## PJ-doc — 48 LLM-generated questions -New corpus (PJ doc, distinct from AVATAR/Neksai). Indexed into Qdrant collection +New corpus (PJ doc, distinct from AVATAR/Contoso). Indexed into Qdrant collection `ir_dev_mvp3_table_summary_v1` from `data/dev/ir_document/pj-doc/chunked-section-table/chunks.json` (174 chunks: 60 section, 24 table, 90 text). @@ -390,7 +390,7 @@ near-perfect on recall. | **Faithfulness jumps 0.92 → 0.96 on 100 Qs vs 40 Qs** | Larger testset has a more balanced mix; multi_hop (the 40Q drag, fa=0.72) is proportionally less represented. Not a regression of the smaller testset. | | **`gpt-4.1-mini` has sporadic multi-second gen stalls** | MVP2-100q had 5 queries at 60–91s gen (vs MVP1-100q max 9.7s same day). Answers correct and short — provider-side tail, not prompt-size. Inflates e2e p95; mean/median unaffected. | | **`gpt-4.1-nano` matches `gpt-4.1-mini` quality at ~1/3 cost** (40 Qs) | Row 07 (mini, MVP2) fa=0.93 ar=0.78 cp=0.81 vs Row 08 (nano, MVP2) fa=0.93 ar=0.81 cp=0.77 — within noise on all RAGAS metrics. Nano tightens e2e p95 from 5556ms → 2243ms (new best in the table). Re-run at 100 Qs recommended before promoting to MVP 1. | -| **PJ-doc is a harder corpus than AVATAR/Neksai on the same stack** | Same `rag-ir-mvp3-table-summary` config: AVATAR 40-Q (proxy: 39-Q rerank-on row, fa≈0.95 historical) vs PJ-doc 48-Q (row 02: fa=0.884, ar=0.706, cp=0.779). ans_rel drops ~0.10 — PJ-doc questions are tougher to answer concisely, even with element-level retrieval at 100%. Multi-hop (7 Qs) and the wider `needs_review` validation rate (16/48) are the likely driver. | +| **PJ-doc is a harder corpus than AVATAR/Contoso on the same stack** | Same `rag-ir-mvp3-table-summary` config: AVATAR 40-Q (proxy: 39-Q rerank-on row, fa≈0.95 historical) vs PJ-doc 48-Q (row 02: fa=0.884, ar=0.706, cp=0.779). ans_rel drops ~0.10 — PJ-doc questions are tougher to answer concisely, even with element-level retrieval at 100%. Multi-hop (7 Qs) and the wider `needs_review` validation rate (16/48) are the likely driver. | | **`element_*` metrics are the right signal on LLM-gen testsets with overlapping chunks** | PJ-doc row 02: `retrieval_recall=0.761` but `element_recall=1.000`, `element_hit=1.000`, `element_mrr@10=0.893`. The LLM generator labels one chunk-id as evidence, but the retriever often returns a sibling chunk containing the same elements. Treat `element_recall`/`element_mrr` as the truth for retrieval health on this kind of testset. | | **Reranker is correctly off by default for inner-loop iteration** (PJ-doc) | Empirically validated by the 39-Q comparison cited in `configs/rag-ir-mvp3-table-summary.yml`: Cohere `rerank-v3.5` adds 6.6× retrieval p95 (649ms → 4300ms) for ~0 quality gain (`element_mrr@10` 0.887 vs 0.889, `substring_match` regressed −0.026). Flip on only for promotion-grade runs. | | **Multi-doc element_recall holds at 0.967 (no per-question doc filter)** | Multi-doc row 02 (PJ 50 + F1 49 = 99 Qs) on the combined corpus, both docs visible to the retriever: `element_recall=0.967`, `element_mrr@10=0.724`. Per-doc: PJ 0.880 / F1 0.898. PJ has 46% cross-doc top-10 leakage (smaller corpus, F1 chunks crowd in) and F1 has 14% — both still find chunks carrying the right source elements ~9/10 times. Cross-doc retrieval is *intentional* here, not a fix-it. The earlier 0.494 element_recall (row 01) was a stale-testset artifact: PJ's `evidence_element_ids` were generated against the API parse and didn't survive the local re-parse (0/77 element_ids matched). Regenerating the PJ testset against the local-parsed chunks fixed the metric without any retrieval changes. **Lesson: re-run testset generation any time the parser changes; chunk_ids stay stable but element_ids do not.** | @@ -400,7 +400,7 @@ near-perfect on recall. | **Indexing-time chunk hygiene removes index pollution** (758 vs 778 prior) | New `IndexingConfig` (`min_section_body_chars=30`, `merge_continuation_sections=True`) drops heading-only sections and merges page-split `(cont.)` siblings before parent exclusion: pj-doc 173 → 165 (8 cont. merged) → 113 indexed; f1-doc 840 → 645 indexed (no cont. sections). Cleaner top-k (no embedding budget on `"10. TAXATION"` or cover-page metadata), and `(cont.)` family dedup now treats split sections as one unit. | | **`exclude_parents_from_index` moved from env var to YAML** (`AppConfig.indexing`) | `Settings.exclude_parents_from_index` (env var `EXCLUDE_PARENTS_FROM_INDEX`) removed in favor of YAML `indexing.exclude_parents_from_index` (default `True`). Configs are now the single source of truth for "how chunks were indexed"; no per-process env to forget. Two new sibling fields land in the same block: `min_section_body_chars` and `merge_continuation_sections`. | | **Enrichment concurrency (max_concurrency=10) is benchmark-neutral** | Multi-doc row 05 (20260504-1511, conc=10) vs row 13 compile log (20260427-0523, conc=1, same 86-Q run): Recall@10 0.937 vs 0.924 (+0.013), MRR@10 0.621 vs 0.641 (−0.020), num_match 0.923 vs 0.897 (+0.026). All deltas within LLM generation variance at temperature=0.1. PR #39 (per-step `batch_as_completed`) correctness confirmed. | -| **ADR 0002 augment-not-replace is benchmark-neutral on entity-prefixed testsets** (Multi-doc 95 Qs) | Multi-doc row 06 (20260505-1054) vs prior 95-Q best (20260504-1603): R@10 0.994 → 0.983 (−0.011), element_recall 1.000 → 0.989 (−0.011), MRR@10 0.723 → 0.724 (+0.001), num_match 0.750 → 0.750 (flat), retrieval p95 618ms → 433ms (−30%). All deltas within loose tolerance. The one flipped question (`qa:0011`: *"What was the cash balance as of December 31, 2023…"*) is a cell-value table query — exactly one of the two failure modes ADR 0002 targets — and regressed from rank 9 → out-of-top-10 because augmenting bodies into all chunks made f1-doc text chunks more competitive on this query, out-ranking the pj-doc table chunk despite its `Cash \| $4,431,280` now being in the dense vector. **Lesson: this LLM-gen testset (auto-question-derived → entity-prefixed) cannot witness the navigational/cell-value wins ADR 0002 targets; the deferred integration regression pins (PR #45 follow-up) are the right gate.** See `experiments/eval/rag-ir-mvp3-all-docs-20260505-1054/fail_cases.md`. | +| **ADR 0002 augment-not-replace is benchmark-neutral on entity-prefixed testsets** (Multi-doc 95 Qs) | Multi-doc row 06 (20260505-1054) vs prior 95-Q best (20260504-1603): R@10 0.994 → 0.983 (−0.011), element_recall 1.000 → 0.989 (−0.011), MRR@10 0.723 → 0.724 (+0.001), num_match 0.750 → 0.750 (flat), retrieval p95 618ms → 433ms (−30%). All deltas within loose tolerance. The one flipped question (`qa:0011`: *"What was the cash balance as of December 31, 2023…"*) is a cell-value table query — exactly one of the two failure modes ADR 0002 targets — and regressed from rank 9 → out-of-top-10 because augmenting bodies into all chunks made f1-doc text chunks more competitive on this query, out-ranking the pj-doc table chunk despite its `Cash \| $3,845,200` now being in the dense vector. **Lesson: this LLM-gen testset (auto-question-derived → entity-prefixed) cannot witness the navigational/cell-value wins ADR 0002 targets; the deferred integration regression pins (PR #45 follow-up) are the right gate.** See `experiments/eval/rag-ir-mvp3-all-docs-20260505-1054/fail_cases.md`. | | **Latest 124-Q row proves the navigational slice is the missing witness** | Multi-doc row 07 (`2026-05-05 14:54 UTC`, `data/dev/ir_document/latest`): legacy categories stay high on recall — direct_lookup `1.000`, semantic_paraphrase `1.000`, table_lookup `0.958`, multi_hop `0.806` — while `navigational` drops to `0.600` R@10 and `0.221` MRR@10. The L2 gate passes cleanly: `0.600 < median(1.000, 1.000, 0.958, 0.806) = 0.979`. Concrete miss cluster: all three `Our Strategy` navigational rows miss their pinned chunk entirely, with `SPECIAL NOTE REGARDING FORWARD-LOOKING STATEMENTS` and TOC-like sections crowding the top ranks instead. See `experiments/eval/rag-ir-mvp3-all-docs-20260505-1454/fail_cases.md`. | --- diff --git a/docs/adr/0002-retrieval-text-augment-not-replace.md b/docs/adr/0002-retrieval-text-augment-not-replace.md index 0c2d2b6..5c7d7d8 100644 --- a/docs/adr/0002-retrieval-text-augment-not-replace.md +++ b/docs/adr/0002-retrieval-text-augment-not-replace.md @@ -31,10 +31,10 @@ both query and document. Our corpus is F-1 / project-document filings. Users ask **navigational** questions like *"how is the corporate structured?"* without naming the issuer. Auto-generated questions all start with the entity name (*"What -strategic corporate restructuring steps did Neksai undertake…"*), so +strategic corporate restructuring steps did Contoso undertake…"*), so entity-less queries miss the dense vector entirely. Issue #43 documents the failure case: `f1-doc:chunk:84` (the actual structural narrative -naming Neksai Inc., Neksai BVI, Neksai SG) never appears in top-10 +naming Contoso Inc., Contoso BVI, Contoso SG) never appears in top-10 for the query *"how is the corporate structured"*, but does for the entity-injected variant *"how is nexksai corporate structured"*. diff --git a/docs/adr/0003-navigational-testset-primary-chunk-evidence.md b/docs/adr/0003-navigational-testset-primary-chunk-evidence.md index 3031126..2a35899 100644 --- a/docs/adr/0003-navigational-testset-primary-chunk-evidence.md +++ b/docs/adr/0003-navigational-testset-primary-chunk-evidence.md @@ -19,8 +19,8 @@ shares: **multiple chunks legitimately satisfy the question**. For *"how is the corporate structured?"* against the f1-doc corpus, the following chunks are all valid retrieval results: -- `f1-doc:chunk:84` — leaf naming the subsidiaries (*Neksai Inc.*, - *Neksai BVI*, *Neksai SG*) +- `f1-doc:chunk:84` — leaf naming the subsidiaries (*Contoso Inc.*, + *Contoso BVI*, *Contoso SG*) - the parent section under *"CORPORATE HISTORY AND STRUCTURE"* — describes the structure narratively - a hypothetical diagram chunk (currently dropped by the parser) — diff --git a/docs/chunk_strategy/section_table.md b/docs/chunk_strategy/section_table.md index 2b39a4e..d36fcb1 100644 --- a/docs/chunk_strategy/section_table.md +++ b/docs/chunk_strategy/section_table.md @@ -339,8 +339,8 @@ paragraphs>"` truncated to `section_content_max_chars`. { "chunk_type": "table", "table_html": "...", - "table_text": "Cash | $4,431,280 | $7,351,900", - "table_text_markdown": "| Cash | $4,431,280 | $7,351,900 |\n| --- | --- | --- |", + "table_text": "Cash | $3,845,200 | $6,120,400", + "table_text_markdown": "| Cash | $3,845,200 | $6,120,400 |\n| --- | --- | --- |", "pre_text": "...", // joined neighbor text (above) "post_text": "...", // joined neighbor text (below) "token_count": @@ -444,9 +444,9 @@ Input elements (post-preprocess): [ {"type": "Title", "text": "1. ORGANIZATION", "element_id": "h1", "metadata": {"category_depth": 0, "page_number": 7}}, {"type": "Title", "text": "(a) Nature of operations","element_id": "h2", "metadata": {"category_depth": 1, "page_number": 7}}, - {"type": "NarrativeText","text": "Neksai Inc. (the \"Company\") was incorporated in Delaware on January 4, 2020.","element_id": "p1", "metadata": {"page_number": 7}}, + {"type": "NarrativeText","text": "Contoso Inc. (the \"Company\") was incorporated in Delaware on March 12, 2019.","element_id": "p1", "metadata": {"page_number": 7}}, {"type": "NarrativeText","text": "(Stated in U.S. Dollars (\"US$\"))","element_id": "p2", "metadata": {"page_number": 8}}, - {"type": "Table", "text": "Cash | $4,431,280 | $7,351,900","element_id": "t1", "metadata": {"page_number": 8, "text_as_html": "
...
"}} + {"type": "Table", "text": "Cash | $3,845,200 | $6,120,400","element_id": "t1", "metadata": {"page_number": 8, "text_as_html": "...
"}} ] ``` @@ -454,7 +454,7 @@ Trace: 1. `Title "1. ORGANIZATION"` (depth 0) → flush (no-op), heading_path = `["1. ORGANIZATION"]`, register parent `:chunk:1`. 2. `Title "(a) Nature of operations"` (depth 1) → flush (no-op), heading_path = `["1. ORGANIZATION", "(a) Nature of operations"]`, register parent `:chunk:2`. -3. `NarrativeText "Neksai Inc. ..."` → append to `active_text_group`. +3. `NarrativeText "Contoso Inc. ..."` → append to `active_text_group`. 4. `NarrativeText "(Stated in U.S. Dollars ("US$"))"` → matches parenthetical-caption-shaped → still treated as text → append to `active_text_group`. *(Note: this only matters when the element type is `Title`/`Header`, the regex check is gated upstream. As a `NarrativeText`, it just goes to the group naturally.)* 5. `Table` → `flush_text_group()` emits `:chunk:3` (text child of parent 2) with the two prose paragraphs joined by `\n`. Then `_build_table_chunk` collects `p2` as `pre_elements` (1 element, less than max=2; stops at Title boundary backward). No `post_elements` (end of input). Emits `:chunk:4` (table child of parent 2). 6. End of input → final `flush_text_group()` no-op. diff --git a/docs/chunker.md b/docs/chunker.md index 11f46e7..d76e68d 100644 --- a/docs/chunker.md +++ b/docs/chunker.md @@ -241,7 +241,7 @@ Table chunks add: `table_html`, `table_text`, `table_text_markdown`, `pre_text`, "chunk_type": "text", "chunk_strategy": "title_hierarchy", "heading_path": ["1. ORGANIZATION", "(a) Nature of operations"], - "content": "1. ORGANIZATION\n(a) Nature of operations\nNeksai Inc. (the \"Company\") was incorporated ...", + "content": "1. ORGANIZATION\n(a) Nature of operations\nContoso Inc. (the \"Company\") was incorporated ...", "source_element_ids": ["t1", "t2", "n1"], "page_number": null, "parent_chunk_id": null @@ -257,13 +257,13 @@ Table chunks add: `table_html`, `table_text`, `table_text_markdown`, `pre_text`, "chunk_type": "table", "chunk_strategy": "section_token", "heading_path": ["CONSOLIDATED BALANCE SHEETS"], - "content": "Section: CONSOLIDATED BALANCE SHEETS\n\nPre-context:\n...\n\nTable:\nCash | $4,431,280 | $7,351,900\n\nPost-context:\n...", + "content": "Section: CONSOLIDATED BALANCE SHEETS\n\nPre-context:\n...\n\nTable:\nCash | $3,845,200 | $6,120,400\n\nPost-context:\n...", "source_element_ids": ["table-element-1"], "page_number": 8, "parent_chunk_id": "avatar_comments_doc:chunk:11", "table_html": "...
", - "table_text": "Cash | $4,431,280 | $7,351,900", - "table_text_markdown": "| Cash | $4,431,280 | $7,351,900 |\n| --- | --- | --- |", + "table_text": "Cash | $3,845,200 | $6,120,400", + "table_text_markdown": "| Cash | $3,845,200 | $6,120,400 |\n| --- | --- | --- |", "pre_text": "", "post_text": "Giving retroactive effect to the issuance of 1 ordinary share..." } diff --git a/docs/command.md b/docs/command.md index b692a54..282ac17 100644 --- a/docs/command.md +++ b/docs/command.md @@ -48,7 +48,7 @@ Example: uv run python -m src.cli convert \ --source docx \ --target pdf \ - --input "data/Pj AVATAR_F page (260205)-with comments.docx" \ + --input "data/ir-sample-a.docx" \ --output data/raw/avatar-comments.pdf ``` @@ -198,7 +198,7 @@ Typical use: Build structured retrieval chunks from normalized parser elements. Detailed chunker behavior and output schema are documented in -[`docs/chunker.md`](/data/lucas/projects/rag/ir_v2/docs/chunker.md). +[`docs/chunker.md`](/docs/chunker.md). Help: @@ -441,7 +441,7 @@ Behavior: Run one-shot baseline RAG over a local `chunks.json` file. Detailed behavior is documented in -[`docs/rag-baseline.md`](/data/lucas/projects/rag/ir_v2/docs/rag-baseline.md). +[`docs/rag-baseline.md`](/docs/rag-baseline.md). Help: diff --git a/docs/enrichment.md b/docs/enrichment.md index 975a12e..e8d0c58 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -60,12 +60,12 @@ enrichment: If `pipeline` is empty, indexing behaves as before and embeds plain chunk `content`. -Ready-made configs live under [`configs/enrichment/`](/data/lucas/projects/rag/ir_v2/configs/enrichment): +Ready-made configs live under [`configs/enrichment/`](/configs/enrichment): -- [`table-context.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/table-context.yml) -- [`auto-keywords.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/auto-keywords.yml) -- [`auto-questions.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/auto-questions.yml) -- [`all.yml`](/data/lucas/projects/rag/ir_v2/configs/enrichment/all.yml) +- [`table-context.yml`](/configs/enrichment/table-context.yml) +- [`auto-keywords.yml`](/configs/enrichment/auto-keywords.yml) +- [`auto-questions.yml`](/configs/enrichment/auto-questions.yml) +- [`all.yml`](/configs/enrichment/all.yml) ## Preview Output @@ -195,9 +195,9 @@ It does not embed text or call the vector store. ## Related Files -- [`xrag/core/enrichment/`](/data/lucas/projects/rag/ir_v2/xrag/core/enrichment) -- [`scripts/enrichment/preview.py`](/data/lucas/projects/rag/ir_v2/scripts/enrichment/preview.py) -- [`scripts/enrichment/show_chunks.py`](/data/lucas/projects/rag/ir_v2/scripts/enrichment/show_chunks.py) -- [`xrag/pipelines/indexing.py`](/data/lucas/projects/rag/ir_v2/xrag/pipelines/indexing.py) -- [`docs/command.md`](/data/lucas/projects/rag/ir_v2/docs/command.md) -- [`docs/sub-plans/ragflow-port-plan.md`](/data/lucas/projects/rag/ir_v2/docs/sub-plans/ragflow-port-plan.md) +- [`xrag/core/enrichment/`](/xrag/core/enrichment) +- [`scripts/enrichment/preview.py`](/scripts/enrichment/preview.py) +- [`scripts/enrichment/show_chunks.py`](/scripts/enrichment/show_chunks.py) +- [`xrag/pipelines/indexing.py`](/xrag/pipelines/indexing.py) +- [`docs/command.md`](/docs/command.md) +- [`docs/sub-plans/ragflow-port-plan.md`](/docs/sub-plans/ragflow-port-plan.md) diff --git a/docs/indexing/table-summary-ablation.md b/docs/indexing/table-summary-ablation.md index 122e25b..fabbf0f 100644 --- a/docs/indexing/table-summary-ablation.md +++ b/docs/indexing/table-summary-ablation.md @@ -120,9 +120,9 @@ Two things conspire: currently-noisy `table_html` (10+ pad columns, split currency cells, repeated multi-row headers). Inspection caught: - `chunk:56`: "hundreds of millions of units" vs. actual $8,810 - (2023) / $438,533 (2024). - - `chunk:129`: "total lease payments of $672.2 million" vs. actual - ~$672K — three orders of magnitude off. + (2023) / $471,280 (2024). + - `chunk:129`: "total lease payments of $540.3 million" vs. actual + ~$540K — three orders of magnitude off. - `chunk:170`: "yen" labels on USD figures. - `chunk:21`: "$0.01 par value" vs. actual $0.00001 (post-subdivision). 2. **Option B drops raw cells from the embed text for tables** — the diff --git a/docs/library.md b/docs/library.md index b3e8e93..d53bea4 100644 --- a/docs/library.md +++ b/docs/library.md @@ -38,7 +38,7 @@ client = Xrag( generation="openai:gpt-4.1-nano", ) -doc = await client.documents.ingest("F1-file.docx", collection="ir_docs") +doc = await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") result = await client.rag.ask( query="What is this document about?", collection="ir_docs", @@ -58,7 +58,7 @@ Runnable example: ```python doc = await client.documents.ingest( - "F1-file.docx", + "ir-sample-b.docx", collection="ir_docs", ) ``` diff --git a/docs/plans.md b/docs/plans.md index 7609143..44451c8 100644 --- a/docs/plans.md +++ b/docs/plans.md @@ -252,7 +252,7 @@ pipeline: artifact_dir: data/artifacts sources: - - path: "data/Pj AVATAR_F page (260205)-with comments.docx" + - path: "data/ir-sample-a.docx" doc_id: avatar_comments_doc parser: diff --git a/docs/rag-baseline.md b/docs/rag-baseline.md index 30ea8f7..ac89786 100644 --- a/docs/rag-baseline.md +++ b/docs/rag-baseline.md @@ -76,7 +76,7 @@ generation: The included config file is: -- [`configs/rag-simple-baseline.yml`](/data/lucas/projects/rag/ir_v2/configs/rag-simple-baseline.yml) +- [`configs/rag-simple-baseline.yml`](/configs/rag-simple-baseline.yml) ## Input @@ -110,7 +110,7 @@ The command prints JSON: { "input_path": "data/artifacts//chunked/chunks.json", "query": "What was Cash as of December 31, 2024?", - "answer": "$7,351,900", + "answer": "$6,120,400", "model": "gpt-4.1-mini", "indexed_chunks": 129, "retrieved_chunks": 5, @@ -122,7 +122,7 @@ The command prints JSON: Code lives in: -- [`xrag/pipelines/rag_baseline.py`](/data/lucas/projects/rag/ir_v2/xrag/pipelines/rag_baseline.py) +- [`xrag/pipelines/rag_baseline.py`](/xrag/pipelines/rag_baseline.py) It uses: diff --git a/docs/sub-plans/chunking-plan.md b/docs/sub-plans/chunking-plan.md index 38f15a5..709824d 100644 --- a/docs/sub-plans/chunking-plan.md +++ b/docs/sub-plans/chunking-plan.md @@ -95,8 +95,8 @@ Practical takeaway: Local references: -- [`docs/dataset-formats.md`](/data/lucas/projects/rag/ir_v2/docs/dataset-formats.md) -- [`docs/survey/datasets.md`](/data/lucas/projects/rag/ir_v2/docs/survey/datasets.md) +- [`docs/dataset-formats.md`](/docs/dataset-formats.md) +- [`docs/survey/datasets.md`](/docs/survey/datasets.md) ## Recommended Chunk Model diff --git a/docs/sub-plans/eval-dataset-generation-plan.md b/docs/sub-plans/eval-dataset-generation-plan.md index 1867806..97d17e4 100644 --- a/docs/sub-plans/eval-dataset-generation-plan.md +++ b/docs/sub-plans/eval-dataset-generation-plan.md @@ -580,7 +580,7 @@ Each final JSONL row should follow a stable schema like: "doc_id": "avatar_comments_doc", "chunk_strategy": "section_table", "question": "What was Cash as of December 31, 2024?", - "answer": "$7,351,900", + "answer": "$6,120,400", "answer_type": "numeric", "reasoning_type": "single_hop_table", "difficulty": "easy", diff --git a/docs/sub-plans/llm-agent-integration.md b/docs/sub-plans/llm-agent-integration.md index 2aa5f34..17c4042 100644 --- a/docs/sub-plans/llm-agent-integration.md +++ b/docs/sub-plans/llm-agent-integration.md @@ -3,7 +3,7 @@ **Status:** draft (rewrite of 2026-04-26 Path-A plan; awaiting Phase-0 kickoff) **Author:** Claude (planning pass, 2026-04-27) **Companion docs:** [`api/apis_design.md`](api/apis_design.md), [`api/phase1.md`](api/phase1.md), [`api/phase1_review.md`](api/phase1_review.md), [`api/system.md`](api/system.md), [`ingest-module-plan.md`](ingest-module-plan.md), [`../../llm-agent/docs/langchain-compat-audit.md`](../../../llm-agent/docs/langchain-compat-audit.md) -**Target consumer repo:** `/data/lucas/projects/rag/llm-agent` (FastAPI + Django ORM + RabbitMQ; Python 3.11; uv) +**Target consumer repo:** `/path/to/llm-agent` (FastAPI + Django ORM + RabbitMQ; Python 3.11; uv) **Engine repo:** `henryle97/xrag` (this repo, ir_v2/; renamed from `henryle97/fin-rag` on 2026-04-27) --- @@ -225,7 +225,7 @@ This test is the gate for tagging v0.1.0. uv build # → dist/xrag-0.1.0-py3-none-any.whl uv venv --python 3.12 /tmp/xrag-test /tmp/xrag-test/bin/uv pip install \ - "/data/lucas/projects/rag/ir_v2/dist/xrag-0.1.0-py3-none-any.whl[rerank-cohere]" + "dist/xrag-0.1.0-py3-none-any.whl[rerank-cohere]" /tmp/xrag-test/bin/python -c "from xrag import run_rag_query, run_ingest_pipeline; print('ok')" ``` @@ -372,7 +372,7 @@ Self-contained, does not piggyback on `EventManagerConfig`. Reproduce the dev-run shape via the new API end-to-end: -1. `POST /api/v1/rag/documents` with `data/raw/ir_document/F1-file.docx` → status reaches `ready` within 2 minutes. +1. `POST /api/v1/rag/documents` with `data/raw/ir_document/ir-sample-b.docx` → status reaches `ready` within 2 minutes. 2. `POST /api/v1/rag/retrievals` with the 87-question testset queries → assert top-line metrics match `experiments/eval/rag-ir-mvp3-all-docs-20260426-1658/summary.txt` within noise: - `deterministic.retrieval_hit ≥ 0.99` - `deterministic.recall_at_10 ≥ 0.97` diff --git a/docs/sub-plans/multi-doc-rag-mvp-plan.md b/docs/sub-plans/multi-doc-rag-mvp-plan.md index 52563c0..e7eddcc 100644 --- a/docs/sub-plans/multi-doc-rag-mvp-plan.md +++ b/docs/sub-plans/multi-doc-rag-mvp-plan.md @@ -335,7 +335,7 @@ Add 5–10 questions the auto-gen pipeline can't produce: full-eval mode. - **`num_match` quality.** It's a substring check, not an LLM judge. It will under-count cases where the model paraphrases - the right number ("$4.4M" vs "4,431,280"). Accept this for + the right number ("$3.8M" vs "3,845,200"). Accept this for the inner loop; full eval at promotion catches it. ### Notes diff --git a/docs/sub-plans/parser-preprocess-plan.md b/docs/sub-plans/parser-preprocess-plan.md index 9fd2e31..3ded850 100644 --- a/docs/sub-plans/parser-preprocess-plan.md +++ b/docs/sub-plans/parser-preprocess-plan.md @@ -88,7 +88,7 @@ Recommended heading heuristic: - repeated 3 or more times - short text, e.g. <= 80 chars - not a substantive sentence - - common company/page-header style content such as `NEKSAI INC.` + - common company/page-header style content such as `CONTOSO INC.` 6. `prefer_table_html(elements)` diff --git a/docs/sub-plans/retrieval-quality-fixes.md b/docs/sub-plans/retrieval-quality-fixes.md index 5fe0136..33f7892 100644 --- a/docs/sub-plans/retrieval-quality-fixes.md +++ b/docs/sub-plans/retrieval-quality-fixes.md @@ -14,8 +14,8 @@ defaults: `gpt-4.1-nano` generator, 3 RAGAS metrics, `gpt-4o-mini` judge). | Trace | Query | Expected | Answer | Primary source rank | |---|---|---|---|---| -| 1 | "total rent expenditure for FY 2023" | US$298,754 | ✅ correct | **1** (lucky) | -| 2 | "total cash by December 31, 2024" | US$7,351,900 | ✅ correct | **8** (balance sheet chunk 12) | +| 1 | "total rent expenditure for FY 2023" | US$341,620 | ✅ correct | **1** (lucky) | +| 2 | "total cash by December 31, 2024" | US$6,120,400 | ✅ correct | **8** (balance sheet chunk 12) | Answers are correct, but the ranking reveals three problems that will bite harder as we drop to smaller top-k, introduce reranking, or move @@ -67,7 +67,7 @@ Two compounding causes: has ~40 rows covering 4 line-item categories. BM25's length normalisation penalises long docs; the dense embedding captures a diffuse "financial statement" topic vector rather than a specific - "cash" signal. The answer row ("Cash | $4,431,280 | $7,351,900") is + "cash" signal. The answer row ("Cash | $3,845,200 | $6,120,400") is one of dozens. Today the generator gets the right answer because a *different* chunk diff --git a/docs/sub-plans/section-table-chunker-review.md b/docs/sub-plans/section-table-chunker-review.md index 66193f8..9a92f50 100644 --- a/docs/sub-plans/section-table-chunker-review.md +++ b/docs/sub-plans/section-table-chunker-review.md @@ -312,8 +312,8 @@ Mapping items in | Bucket | Item | Chunker contribution | |---|---|---| | C — wrong-cell (×14) | `ETH 2023 = 290` | S3: gold in chunk:120 (cont.), retrieval saturated by chunks 116-118 under base parent | -| C — wrong-cell | `salary = $(361,993)` | S3 + S2: same fragmentation; nearby paragraph emitted 3× crowds top-k | -| D — retrieval miss (×3) | `digital assets 2023 = $5,868,425` (gold chunk:12 in balance sheet) | S3: section 114+120 carry strong "digital assets" lexical match and out-rank the balance-sheet line | +| C — wrong-cell | `salary = $(405,760)` | S3 + S2: same fragmentation; nearby paragraph emitted 3× crowds top-k | +| D — retrieval miss (×3) | `digital assets 2023 = $6,214,730` (gold chunk:12 in balance sheet) | S3: section 114+120 carry strong "digital assets" lexical match and out-rank the balance-sheet line | | E — found at low rank (×16) | `cash 2024` etc. | S1 + S6: parent/sibling near-duplicates eat top-3 slots, evidence pushed to rank 3-7 | S3 (`(cont.)` fragmentation) is the single highest-leverage finding — diff --git a/docs/sub-plans/xrag-public-api.md b/docs/sub-plans/xrag-public-api.md index 67bd65e..828a2e8 100644 --- a/docs/sub-plans/xrag-public-api.md +++ b/docs/sub-plans/xrag-public-api.md @@ -106,7 +106,7 @@ Concrete patterns observed in current docs (April 2026), via Context7. Survey is from xrag import Xrag client = Xrag.from_env() -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") result = await client.rag.ask("What's the FY-23 revenue?", collection="ir_docs") print(result.answer) ``` @@ -173,14 +173,14 @@ Only `qdrant_url` and `embedding` are strictly required. Everything else has sen ```python # Single-tenant prototype — implicit "default" tenant -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") # Explicit per-call (multi-tenant from a request handler) -await client.documents.ingest("F1-file.docx", collection="ir_docs", tenant_id="dev") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs", tenant_id="dev") # Bound tenant for a sequence of calls (request-scoped) tenant = client.for_tenant("dev") # XragTenant — tenant_id pre-bound -await tenant.documents.ingest("F1-file.docx", collection="ir_docs") +await tenant.documents.ingest("ir-sample-b.docx", collection="ir_docs") await tenant.retrievals.search(query="...", collection="ir_docs", top_k=10) ``` @@ -484,7 +484,7 @@ User-supplied `artifacts_dir` paths can be reused across calls — that's the wh "hostname": "ingest-worker-3", "started_at": "2026-04-27T08:14:22.481Z", "document_id": "doc_01HXY...", - "source_name": "F1-file.docx", + "source_name": "ir-sample-b.docx", "xrag_version": "0.2.0" } ``` @@ -665,7 +665,7 @@ client = Xrag( ```python # ingest as usual await client.documents.ingest( - "F1-file.docx", collection="ir_docs", tenant_id="dev", + "ir-sample-b.docx", collection="ir_docs", tenant_id="dev", ) # retrieve, then generate with whatever you want @@ -699,7 +699,7 @@ What the caller controls themselves: prompt, model choice, temperature, structur from xrag import XragSync client = XragSync.from_env() -doc = client.documents.ingest("F1-file.docx", collection="ir_docs", tenant_id="dev") +doc = client.documents.ingest("ir-sample-b.docx", collection="ir_docs", tenant_id="dev") print(client.rag.ask(query="...", collection="ir_docs", tenant_id="dev").answer) ``` @@ -936,7 +936,7 @@ query_result = run_rag("What's the revenue?", cfg) from xrag import Xrag client = Xrag.from_env() -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") result = await client.rag.ask(query="What's the revenue?", collection="ir_docs") ``` @@ -945,7 +945,7 @@ result = await client.rag.ask(query="What's the revenue?", collection="ir_docs") from xrag import Xrag client = Xrag(qdrant_url="http://qdrant:6333", embedding="openai:text-embedding-3-small") -await client.documents.ingest("F1-file.docx", collection="ir_docs") +await client.documents.ingest("ir-sample-b.docx", collection="ir_docs") search = await client.retrievals.search( query="What's the revenue?", collection="ir_docs", top_k=10, ) diff --git a/docs/survey/SOTA-RAG-for-financial-documents.md b/docs/survey/SOTA-RAG-for-financial-documents.md index e8b406f..bb3da8c 100644 --- a/docs/survey/SOTA-RAG-for-financial-documents.md +++ b/docs/survey/SOTA-RAG-for-financial-documents.md @@ -424,4 +424,4 @@ When you're ready for multi-company (Level 2), you download Microsoft and Tesla **You never index FinanceBench itself.** It's purely evaluation data — like how you never put your unit test assertions into your application database. -Does that clear it up? \ No newline at end of file +Does that clear it up? diff --git a/docs/survey/claude-rag-survey.md b/docs/survey/claude-rag-survey.md index d3680a8..9a5bc98 100644 --- a/docs/survey/claude-rag-survey.md +++ b/docs/survey/claude-rag-survey.md @@ -11,7 +11,7 @@ Building a RAG system for financial reports (DOCX/PDF) with dense text + many ta ## CHALLENGE 1: Document Parsing & Table Extraction -**Problem**: Financial reports like the Neksai Inc. filing contain 20+ complex tables (balance sheets, income statements, cash flows, footnote tables) interleaved with dense narrative text. Standard text extraction destroys table structure, making retrieval of exact numbers impossible. +**Problem**: Financial reports like the Contoso Inc. filing contain 20+ complex tables (balance sheets, income statements, cash flows, footnote tables) interleaved with dense narrative text. Standard text extraction destroys table structure, making retrieval of exact numbers impossible. ### Solution A: MinerU + HTML Table Preservation @@ -51,7 +51,7 @@ Step 1.6 Validate extraction against golden samples: ## CHALLENGE 2: Table-Aware Chunking Strategy -**Problem**: Standard fixed-size chunking (400–512 tokens) splits tables mid-row, destroying the relationship between row headers and values. A question like "What was Neksai's total current assets in 2024?" requires the entire balance sheet table to be in one chunk, but that table may be 800+ tokens. +**Problem**: Standard fixed-size chunking (400–512 tokens) splits tables mid-row, destroying the relationship between row headers and values. A question like "What was Contoso's total current assets in 2024?" requires the entire balance sheet table to be in one chunk, but that table may be 800+ tokens. ### Solution A: Structure-Aware Chunking with Table Atomization @@ -76,9 +76,9 @@ Chunk types: ``` Parent: Full "Balance Sheet" section (table + surrounding text) - └─ Child: "Total current assets: $13,387,879 (Dec 31, 2024)" - └─ Child: "Cash: $7,351,900 (Dec 31, 2024)" - └─ Child: "Digital assets: $2,853,590 (Dec 31, 2024)" + └─ Child: "Total current assets: $15,640,290 (Dec 31, 2024)" + └─ Child: "Cash: $6,120,400 (Dec 31, 2024)" + └─ Child: "Digital assets: $3,106,470 (Dec 31, 2024)" ``` - **Pros**: Precise retrieval + complete context, solves the fundamental chunk-size tradeoff (RAGFlow's "structural conflict"), enables multi-hop reasoning @@ -99,11 +99,11 @@ Step 2.1 Define chunk hierarchy: Step 2.2 For each table, generate BOTH: a) Full table as HTML (parent chunk) b) Row-group summaries as natural language (child chunks): - "As of December 31, 2024, Neksai's cash was $7,351,900 and - digital assets were $2,853,590" + "As of December 31, 2024, Contoso's cash was $6,120,400 and + digital assets were $3,106,470" Step 2.3 Add contextual metadata to every chunk: { - "doc_id": "neksai_10k_2024", + "doc_id": "contoso_10k_2024", "section": "Consolidated Balance Sheets", "page": 3, "chunk_type": "table", @@ -120,7 +120,7 @@ Step 2.5 Validate: for each gold Q&A pair, verify the answer exists in at ## CHALLENGE 3: Hybrid Retrieval + Reranking -**Problem**: Dense vector search alone misses exact financial terms (e.g., "PCAOB ID:1171", "ASU 2023-08", specific dollar amounts). Sparse search alone misses semantic paraphrases ("profit" vs. "net income"). Financial queries require both. +**Problem**: Dense vector search alone misses exact financial terms (e.g., "PCAOB ID:9999", "ASU 2023-08", specific dollar amounts). Sparse search alone misses semantic paraphrases ("profit" vs. "net income"). Financial queries require both. ### Solution A: BM25 + Dense Vectors + Reciprocal Rank Fusion (RRF) @@ -234,10 +234,10 @@ Step 4.1 Design generation prompt with: - Uncertainty handling ("INSUFFICIENT_CONTEXT") - Output format: {answer, citations[], confidence, reasoning} Step 4.2 Implement answer type detection: - - EXACT_VALUE: "What was total revenue in 2024?" → $45,042,408 + - EXACT_VALUE: "What was total revenue in 2024?" → $48,715,220 - COMPARISON: "How did revenue change YoY?" → calculation + values - - EXPLANATION: "What is Neksai's revenue recognition policy?" → text - - YES_NO: "Does Neksai have deferred tax assets?" → Yes/No + evidence + - EXPLANATION: "What is Contoso's revenue recognition policy?" → text + - YES_NO: "Does Contoso have deferred tax assets?" → Yes/No + evidence Step 4.3 For EXACT_VALUE questions: - Extract the number from the answer - Regex-validate it exists in a retrieved chunk @@ -351,19 +351,19 @@ Evaluation layers: ### Implementation Steps ``` -Step 6.1 Build gold QA dataset from the Neksai financial statements: +Step 6.1 Build gold QA dataset from the Contoso financial statements: Category 1 — Exact Value (deterministic): - Q: "What was Neksai's net income for 2024?" - A: "$5,741,797" + Q: "What was Contoso's net income for 2024?" + A: "$6,003,510" Gold chunks: [balance_sheet_2024, income_statement_2024] Category 2 — Calculation: Q: "What was the gross profit margin in 2024?" - A: "64.1% ($28,851,979 / $45,042,408)" + A: "64.1% ($31,402,650 / $48,715,220)" Gold chunks: [income_statement_2024] Category 3 — Explanation (LLM-judged): - Q: "How does Neksai recognize revenue from subscription services?" + Q: "How does Contoso recognize revenue from subscription services?" A: "Over time within the subscription period..." Gold chunks: [note_2_revenue_recognition] @@ -374,7 +374,7 @@ Step 6.1 Build gold QA dataset from the Neksai financial statements: Gold chunks: [note_1_reorganization] Category 5 — Unanswerable: - Q: "What is Neksai's projected revenue for 2025?" + Q: "What is Contoso's projected revenue for 2025?" A: "INSUFFICIENT_CONTEXT" Gold chunks: [] (nothing should be relevant) @@ -488,7 +488,7 @@ Step 6.5 Private-set robustness measures: ### Phase 1: Foundation (Weeks 1-2) | Step | Task | Deliverable | |------|------|-------------| -| 1.1 | Set up MinerU + Docling parsing | Parsed Neksai report with 95%+ table accuracy | +| 1.1 | Set up MinerU + Docling parsing | Parsed Contoso report with 95%+ table accuracy | | 1.2 | Build multi-granularity chunker | Parent/child chunk pairs with metadata | | 1.3 | Set up vector store + BM25 index | Indexed chunks, searchable | | 1.4 | Basic retrieval pipeline (dense only) | Baseline Recall@5 measurement | @@ -559,4 +559,4 @@ Step 6.5 Private-set robustness measures: - **GaRAGe** (ACL 2025) — Grounding annotations benchmark for RAG evaluation - **RAGAS** — Automated RAG evaluation framework - **MinerU** (OpenDataLab) — 97.5 mAP layout detection, best table extraction -- **Docling** (IBM) — MIT-licensed, TableFormer-powered parsing \ No newline at end of file +- **Docling** (IBM) — MIT-licensed, TableFormer-powered parsing diff --git a/docs/survey/datasets.md b/docs/survey/datasets.md index ca3ced8..fc7046c 100644 --- a/docs/survey/datasets.md +++ b/docs/survey/datasets.md @@ -1,6 +1,6 @@ # Datasets -- **FinanceBench** +- **FinanceBench** - **T2-RAGBench** - FinDER Here's the landscape of RAG evaluation datasets, organized from most to least relevant for your financial report use case: @@ -41,6 +41,6 @@ Here's the landscape of RAG evaluation datasets, organized from most to least re **FiQA** (2018) — opinion-based financial QA, mainly sentiment. **ConvFinQA** (2022) — conversational multi-turn financial QA. **HotpotQA** — general multi-hop, not financial. **NaturalQuestions** — general single-hop from Google search. -## What to use for your Neksai project +## What to use for your Contoso project Start with **FinanceBench** as your primary reference for format and question style — its evidence provenance schema (evidence_text + page + document) directly inspired your golden dataset design. Use **T2-RAGBench** or **FinQA** as a secondary reference for table-heavy numerical reasoning evaluation. For multi-hop testing, look at **FinDER** for realistic analyst-style queries. Your custom dataset fills the gap that none of these cover: single-company, full-report RAG with deterministic + LLM-judge hybrid evaluation and mandatory telemetry scoring. diff --git a/docs/survey/grok-rag-survey.md b/docs/survey/grok-rag-survey.md index e2471b8..6e735b0 100644 --- a/docs/survey/grok-rag-survey.md +++ b/docs/survey/grok-rag-survey.md @@ -61,6 +61,6 @@ For a deployable system today (English SEC/earnings reports): - Add post-retrieval re-ranking and citation/attribution (cell-level in HierFinRAG). - For scale: Binary quantization + hybrid search reduces costs dramatically. -These approaches (especially HierFinRAG + Unstructured.io backbone) represent the current English-language SOTA for reliable, hallucination-resistant RAG on text + table financial reports. They move far beyond vanilla RAG by explicitly modeling the interplay between narrative and numerical data. +These approaches (especially HierFinRAG + Unstructured.io backbone) represent the current English-language SOTA for reliable, hallucination-resistant RAG on text + table financial reports. They move far beyond vanilla RAG by explicitly modeling the interplay between narrative and numerical data. -If you need code snippets, a specific implementation guide (e.g., LlamaIndex + Unstructured + HierFinRAG components), or evaluation on a sample 10-K, let me know! \ No newline at end of file +If you need code snippets, a specific implementation guide (e.g., LlamaIndex + Unstructured + HierFinRAG components), or evaluation on a sample 10-K, let me know! diff --git a/docs/survey/preprocessing-and-normalization.md b/docs/survey/preprocessing-and-normalization.md index 283acb8..2872e42 100644 --- a/docs/survey/preprocessing-and-normalization.md +++ b/docs/survey/preprocessing-and-normalization.md @@ -130,8 +130,8 @@ The largest ROI lever for financial documents. headers ("Year ended December 31" spanning 2024/2023). - **Empty row / column trimming** — visual spacers between sections produce all-empty rows; drop them. -- **Currency-cell merging** — accounting style `$ | 4,431,280` should - become a single cell `$4,431,280`; otherwise the `$` becomes its own +- **Currency-cell merging** — accounting style `$ | 3,845,200` should + become a single cell `$3,845,200`; otherwise the `$` becomes its own noise column. (This repo's `merge_table_currency_cells` step.) - **Section-row preservation** — `ASSETS`, `Current assets:` headers should remain as anchor rows, not be dropped as "empty after the @@ -328,7 +328,7 @@ Specific failure modes preprocessing should improve: because heading was a `UncategorizedText`. Caught by recall drop on questions whose answers live in body content but are heading-disambiguated. -- **Table currency mis-segmentation** — `$ | 4,431,280` → two cells → +- **Table currency mis-segmentation** — `$ | 3,845,200` → two cells → retriever returns either the `$` cell or the number cell; generator hallucinates the missing half. Caught by num_match drops on numeric-answer questions. diff --git a/examples/01_quickstart.py b/examples/01_quickstart.py index 3551b54..c2c03ad 100644 --- a/examples/01_quickstart.py +++ b/examples/01_quickstart.py @@ -15,7 +15,7 @@ from xrag import Xrag -DOC_PATH = Path("data/raw/ir_document/F1-file.docx") +DOC_PATH = Path("data/raw/ir_document/ir-sample-b.docx") COLLECTION = "ir_quickstart" diff --git a/examples/04_multi_tenant.py b/examples/04_multi_tenant.py index 1154af7..235c6aa 100644 --- a/examples/04_multi_tenant.py +++ b/examples/04_multi_tenant.py @@ -11,8 +11,8 @@ from xrag import Xrag -DOC_A = Path("data/raw/ir_document/F1-file.docx") -DOC_B = Path("data/raw/ir_document/Pj_AVATAR_F_page_260205.docx") +DOC_A = Path("data/raw/ir_document/ir-sample-b.docx") +DOC_B = Path("data/raw/ir_document/ir-sample-a.docx") COLLECTION = "ir_multitenant" diff --git a/examples/08_per_call_overrides.py b/examples/08_per_call_overrides.py index b421323..4bb66b7 100644 --- a/examples/08_per_call_overrides.py +++ b/examples/08_per_call_overrides.py @@ -12,7 +12,7 @@ from xrag import Xrag -DOC_PATH = Path("data/raw/ir_document/F1-file.docx") +DOC_PATH = Path("data/raw/ir_document/ir-sample-b.docx") COLLECTION = "ir_overrides" diff --git a/examples/09_parse_docx_api_vs_local.py b/examples/09_parse_docx_api_vs_local.py index 9b8a9c4..5394e50 100644 --- a/examples/09_parse_docx_api_vs_local.py +++ b/examples/09_parse_docx_api_vs_local.py @@ -25,7 +25,7 @@ from xrag.config.models import AppConfig, PipelineConfig, ProviderConfig, SourceConfig from xrag.pipelines.parser import run_parser_pipeline -DOC_PATH = Path("data/raw/ir_document/F1-file.docx") +DOC_PATH = Path("data/raw/ir_document/ir-sample-b.docx") ROOT = Path("data/dev/xrag_examples/09_parse_docx_api_vs_local") diff --git a/pyproject.toml b/pyproject.toml index e616dc9..d792d28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,8 +93,6 @@ select = ["E", "F", "I", "UP", "B"] "tools/review/chunker_review.py" = ["E501"] "tools/eval_dataset/review.py" = ["E501"] "tools/evaluation/preview.py" = ["E501"] -# news.py is a standalone script, not part of the package -"news.py" = ["E501", "B904"] [tool.ruff.format] quote-style = "double" diff --git a/scripts/ingest/ir_latest.sh b/scripts/ingest/ir_latest.sh index 0a8f158..60abc4d 100755 --- a/scripts/ingest/ir_latest.sh +++ b/scripts/ingest/ir_latest.sh @@ -17,10 +17,10 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$REPO_ROOT" CONFIG="configs/rag-ir-latest.yml" -RAW_DIR="/data/lucas/projects/rag/ir_v2/data/dev/ir_document/raw" -OUTPUT_DIR="/data/lucas/projects/rag/ir_v2/data/dev/ir_document/latest" -F1_DOC="$RAW_DIR/F1-file.docx" -PJ_DOC="$RAW_DIR/Pj_AVATAR_F_page_260205.docx" +RAW_DIR="data/dev/ir_document/raw" +OUTPUT_DIR="data/dev/ir_document/latest" +F1_DOC="$RAW_DIR/ir-sample-b.docx" +PJ_DOC="$RAW_DIR/ir-sample-a.docx" if [[ ! -f "$CONFIG" ]]; then echo "ERROR: config not found: $CONFIG" >&2 diff --git a/scripts/parser/parse_f1_doc.sh b/scripts/parser/parse_f1_doc.sh index 565db4c..2ec01f9 100644 --- a/scripts/parser/parse_f1_doc.sh +++ b/scripts/parser/parse_f1_doc.sh @@ -128,4 +128,3 @@ PY # uv run python -m xrag.cli parser \ # --config data/dev/ir_document/configs/parsers/parse-f1-doc.yml - diff --git a/scripts/parser/parse_pj_doc.sh b/scripts/parser/parse_pj_doc.sh index d124ffd..f6e50e9 100644 --- a/scripts/parser/parse_pj_doc.sh +++ b/scripts/parser/parse_pj_doc.sh @@ -126,4 +126,3 @@ PY # uv run python -m xrag.cli parser \ # --config data/dev/ir_document/configs/parsers/parse-f1-doc.yml - diff --git a/tests/unit/test_chunk_prepare.py b/tests/unit/test_chunk_prepare.py index 87b4722..6e88b02 100644 --- a/tests/unit/test_chunk_prepare.py +++ b/tests/unit/test_chunk_prepare.py @@ -36,13 +36,13 @@ def test_run_chunk_prepare_pipeline_builds_section_text_and_table_chunks(tmp_pat }, { "type": "Table", - "text": "Cash | $4,431,280 | $7,351,900", + "text": "Cash | $3,845,200 | $6,120,400", "element_id": "table-1", "metadata": { "page_number": 3, "text_as_html": ( - "" - "
Cash$4,431,280$7,351,900
" + "" + "
Cash$3,845,200$6,120,400
" ), }, }, @@ -76,9 +76,9 @@ def test_run_chunk_prepare_pipeline_builds_section_text_and_table_chunks(tmp_pat assert text_chunk["parent_id"] == section_chunk["id"] assert table_chunk["parent_id"] == section_chunk["id"] assert table_chunk["metadata"]["table_html"].startswith("") - assert table_chunk["metadata"]["table_text"] == "Cash | $4,431,280 | $7,351,900" + assert table_chunk["metadata"]["table_text"] == "Cash | $3,845,200 | $6,120,400" assert table_chunk["metadata"]["table_text_markdown"] == ( - "| Cash | $4,431,280 | $7,351,900 |\n| --- | --- | --- |" + "| Cash | $3,845,200 | $6,120,400 |\n| --- | --- | --- |" ) assert ( "The balance sheets present the financial position." in table_chunk["metadata"]["pre_text"] @@ -153,13 +153,13 @@ def test_run_chunk_prepare_pipeline_supports_section_token_strategy(tmp_path: Pa }, { "type": "Table", - "text": "Cash | $4,431,280 | $7,351,900", + "text": "Cash | $3,845,200 | $6,120,400", "element_id": "table-1", "metadata": { "page_number": 3, "text_as_html": ( - "
" - "
Cash$4,431,280$7,351,900
" + "" + "
Cash$3,845,200$6,120,400
" ), }, }, @@ -196,7 +196,7 @@ def test_run_chunk_prepare_pipeline_supports_section_token_strategy(tmp_path: Pa assert table_chunk["metadata"]["chunk_strategy"] == "section_token" assert table_chunk["parent_id"] == section_chunk["id"] assert table_chunk["metadata"]["table_text_markdown"].startswith( - "| Cash | $4,431,280 | $7,351,900 |" + "| Cash | $3,845,200 | $6,120,400 |" ) assert table_chunk["metadata"]["pre_text"] assert table_chunk["metadata"]["post_text"] @@ -323,13 +323,13 @@ def test_section_table_chunker_skips_parenthetical_caption_as_heading( }, { "type": "Table", - "text": "Cash | 4,431,280 | 7,351,900", + "text": "Cash | 3,845,200 | 6,120,400", "element_id": "table-1", "metadata": { "page_number": 3, "text_as_html": ( - "" - "
Cash4,431,2807,351,900
" + "" + "
Cash3,845,2006,120,400
" ), }, }, diff --git a/tests/unit/test_embed_text.py b/tests/unit/test_embed_text.py index 3ad4e8d..c73593c 100644 --- a/tests/unit/test_embed_text.py +++ b/tests/unit/test_embed_text.py @@ -89,7 +89,7 @@ def test_table_chunk_with_summary_and_questions_orders_table_text_first() -> Non """ chunk = { "chunk_type": "table", - "table_text": "Cash | $4,431,280 | $7,351,900", + "table_text": "Cash | $3,845,200 | $6,120,400", "summary_kwd": "Balance sheet snapshot of cash position year over year.", "question_kwd": [ "What was Cash on the December 2025 balance sheet?", @@ -99,7 +99,7 @@ def test_table_chunk_with_summary_and_questions_orders_table_text_first() -> Non } assert compose_embed_text(chunk) == ( - "Cash | $4,431,280 | $7,351,900\n\n" + "Cash | $3,845,200 | $6,120,400\n\n" "Balance sheet snapshot of cash position year over year.\n\n" "Questions:\n" "What was Cash on the December 2025 balance sheet?\n" diff --git a/tests/unit/test_enrichment.py b/tests/unit/test_enrichment.py index 79c072c..69cb8da 100644 --- a/tests/unit/test_enrichment.py +++ b/tests/unit/test_enrichment.py @@ -53,8 +53,8 @@ def test_run_enrichment_pipeline_applies_ragflow_fields(monkeypatch) -> None: "chunk_id": "c2", "chunk_type": "table", "heading_path": ["CONSOLIDATED BALANCE SHEETS"], - "content": "Cash 4,431,280 7,351,900", - "table_text": "Cash | 4,431,280 | 7,351,900", + "content": "Cash 3,845,200 6,120,400", + "table_text": "Cash | 3,845,200 | 6,120,400", }, { "chunk_id": "c3", diff --git a/tests/unit/test_eval_create_dataset.py b/tests/unit/test_eval_create_dataset.py index 720652e..f281896 100644 --- a/tests/unit/test_eval_create_dataset.py +++ b/tests/unit/test_eval_create_dataset.py @@ -18,8 +18,8 @@ _SIMPLE_TABLE_HTML = ( "" "" - "" - "" + "" + "" "
December 31, 2023December 31, 2024
Cash$4,431,280$7,351,900
Digital assets$317,662$1,012,543
Cash$3,845,200$6,120,400
Digital assets$352,940$1,012,543
" ) @@ -42,7 +42,7 @@ def _build_table_chunk( "page_number": page_number, "parent_chunk_id": "test_doc:chunk:1", "table_html": table_html, - "table_text": "Cash | $4,431,280 | $7,351,900", + "table_text": "Cash | $3,845,200 | $6,120,400", "pre_text": "The balance sheets present the financial position.", "post_text": "Cash increased year over year.", } @@ -167,7 +167,7 @@ def test_parse_table_html_extracts_grid() -> None: assert "Cash" in grid.row_labels assert "Digital assets" in grid.row_labels assert len(grid.values) == 2 - assert any("4,431,280" in v for v in grid.values[0]) + assert any("3,845,200" in v for v in grid.values[0]) def test_parse_table_html_returns_none_for_empty() -> None: diff --git a/tests/unit/test_eval_dataset_review.py b/tests/unit/test_eval_dataset_review.py index a22d4a8..1bad6f4 100644 --- a/tests/unit/test_eval_dataset_review.py +++ b/tests/unit/test_eval_dataset_review.py @@ -19,7 +19,7 @@ def _sample_items() -> list[dict]: { "example_id": "doc:qa:0001", "question": "What was total cash as of Dec 31?", - "answer": "$7,351,900", + "answer": "$6,120,400", "answer_type": "numeric", "reasoning_type": "table_lookup", "difficulty": "easy", @@ -123,7 +123,7 @@ def test_load_chunks_index_handles_canonical_envelope(tmp_path: Path) -> None: { "id": "doc:chunk:12", "doc_id": "doc", - "text": "Cash at Dec 31 was 7351900.", + "text": "Cash at Dec 31 was 6120400.", "metadata": { "heading_path": ["Balance Sheet"], }, @@ -138,7 +138,7 @@ def test_load_chunks_index_handles_canonical_envelope(tmp_path: Path) -> None: index = load_chunks_index(chunks_path) - assert index["doc:chunk:12"]["content"] == "Cash at Dec 31 was 7351900." + assert index["doc:chunk:12"]["content"] == "Cash at Dec 31 was 6120400." assert index["doc:chunk:12"]["heading_path"] == ["Balance Sheet"] @@ -165,7 +165,7 @@ def test_generate_dataset_review_html_with_chunks_resolves_evidence() -> None: chunks_by_id = { "doc:chunk:12": { "chunk_id": "doc:chunk:12", - "content": "Cash at Dec 31 was 7351900.", + "content": "Cash at Dec 31 was 6120400.", "heading_path": ["Balance Sheet"], "chunk_type": "table", } @@ -173,7 +173,7 @@ def test_generate_dataset_review_html_with_chunks_resolves_evidence() -> None: html_out = generate_dataset_review_html(items, chunks_by_id=chunks_by_id) # The chunk content must be embedded somewhere so the JS can resolve it - assert "Cash at Dec 31 was 7351900." in html_out + assert "Cash at Dec 31 was 6120400." in html_out assert "Balance Sheet" in html_out @@ -207,7 +207,7 @@ def test_eval_review_cli_with_chunks(tmp_path: Path) -> None: "chunks": [ { "chunk_id": "doc:chunk:12", - "content": "Cash at Dec 31 was 7351900.", + "content": "Cash at Dec 31 was 6120400.", "heading_path": ["Balance Sheet"], "chunk_type": "table", } @@ -234,4 +234,4 @@ def test_eval_review_cli_with_chunks(tmp_path: Path) -> None: assert result.exit_code == 0, result.stdout rendered = output_path.read_text() - assert "Cash at Dec 31 was 7351900." in rendered + assert "Cash at Dec 31 was 6120400." in rendered diff --git a/tests/unit/test_eval_run.py b/tests/unit/test_eval_run.py index 7fee505..93aab0a 100644 --- a/tests/unit/test_eval_run.py +++ b/tests/unit/test_eval_run.py @@ -126,7 +126,7 @@ def _build_testset_jsonl(tmp_path: Path) -> Path: "example_id": "doc1:qa:0001", "doc_id": "doc1", "question": "What was Cash as of 2024?", - "answer": "$7,351,900", + "answer": "$6,120,400", "answer_type": "numeric", "reasoning_type": "single_hop_table", "difficulty": "easy", @@ -226,10 +226,10 @@ def test_deterministic_metrics_numeric_match() -> None: q = EvalQuestion( question_id="q1", question="Q?", - ground_truth="$4,431,280", + ground_truth="$3,845,200", answer_type="numeric", ) - metrics = _compute_deterministic_metrics(q, "The value was 4431280 dollars.", []) + metrics = _compute_deterministic_metrics(q, "The value was 3845200 dollars.", []) assert metrics["numeric_match"] == 1.0 @@ -248,7 +248,7 @@ def test_deterministic_metrics_no_evidence_skips_retrieval() -> None: def test_extract_number_handles_formatting() -> None: """Extract number from various text formats.""" - assert _extract_number("$4,431,280") == 4431280.0 + assert _extract_number("$3,845,200") == 3845200.0 assert _extract_number("about 3.5 million") == 3.5 assert _extract_number("no number here") is None diff --git a/tests/unit/test_llm_dataset_generator.py b/tests/unit/test_llm_dataset_generator.py index f78769a..0fa80a6 100644 --- a/tests/unit/test_llm_dataset_generator.py +++ b/tests/unit/test_llm_dataset_generator.py @@ -67,13 +67,13 @@ def _make_table_chunk(chunk_id: str = "test_doc:chunk:10") -> dict[str, Any]: return _make_chunk( chunk_id=chunk_id, chunk_type="table", - content="Cash | $4,431,280 | $7,351,900", + content="Cash | $3,845,200 | $6,120,400", heading_path=["CONSOLIDATED BALANCE SHEETS"], table_html=( "" "" - "" - "" + "" + "" "
December 31, 2023December 31, 2024
Cash$4,431,280$7,351,900
Digital assets$317,662$1,012,543
Cash$3,845,200$6,120,400
Digital assets$352,940$1,012,543
" ), ) @@ -150,21 +150,21 @@ def _mock_direct_lookup_response(chunk_ids: list[str]) -> DirectLookupResponse: def _mock_table_lookup_response(chunk_id: str) -> DirectLookupResponse: """Canned response for table_lookup — answers verbatim from the test - table chunk's `table_html` ($4,431,280 and $7,351,900).""" + table chunk's `table_html` ($3,845,200 and $6,120,400).""" return DirectLookupResponse( questions=[ GeneratedQA( - verbatim_span="$4,431,280", + verbatim_span="$3,845,200", question="What was cash as of December 31, 2023?", - answer="$4,431,280", + answer="$3,845,200", answer_type="numeric", difficulty="easy", evidence_chunk_id=chunk_id, ), GeneratedQA( - verbatim_span="$7,351,900", + verbatim_span="$6,120,400", question="What was cash as of December 31, 2024?", - answer="$7,351,900", + answer="$6,120,400", answer_type="numeric", difficulty="easy", evidence_chunk_id=chunk_id, @@ -634,24 +634,24 @@ def mock_invoke(prompt: str) -> Any: def test_answer_is_grounded_numeric_exact_match() -> None: """Verbatim numeric answer is grounded.""" - chunk = "Net cash provided by operating activities was $12,945,449 in 2023." - assert _answer_is_grounded("$12,945,449", chunk, "numeric") is True + chunk = "Net cash provided by operating activities was $14,203,880 in 2023." + assert _answer_is_grounded("$14,203,880", chunk, "numeric") is True def test_answer_is_grounded_numeric_rejects_magnitude_rounding() -> None: """The whole point of the strict check: '$13 million' is NOT grounded - in a chunk that says '$12,945,449' — they differ by far more than the + in a chunk that says '$14,203,880' — they differ by far more than the 0.5 % tolerance the validator uses.""" - chunk = "Net cash provided by operating activities was $12,945,449 in 2023." + chunk = "Net cash provided by operating activities was $14,203,880 in 2023." assert _answer_is_grounded("$13 million", chunk, "numeric") is False assert _answer_is_grounded("$1 million", chunk, "numeric") is False def test_answer_is_grounded_numeric_tolerates_minor_formatting() -> None: """Tiny formatting differences (within 0.5 %) still count as grounded.""" - chunk = "Cash 7,351,900 at end of year" - # 7,351,900 → answer of 7351900 (no commas) should still match - assert _answer_is_grounded("7351900", chunk, "numeric") is True + chunk = "Cash 6,120,400 at end of year" + # 6,120,400 → answer of 6120400 (no commas) should still match + assert _answer_is_grounded("6120400", chunk, "numeric") is True def test_answer_is_grounded_string_substring() -> None: @@ -681,7 +681,7 @@ def test_validator_rejects_ungrounded_numeric_answers( "doc_id": "test_doc", "chunk_type": "text", "chunk_strategy": "section_table", - "content": "Net cash provided by operating activities was $12,945,449 in 2023. " * 3, + "content": "Net cash provided by operating activities was $14,203,880 in 2023. " * 3, "heading_path": ["Cash flow"], "child_chunk_ids": [], "source_element_ids": ["e1"], @@ -698,9 +698,9 @@ def test_validator_rejects_ungrounded_numeric_answers( evidence_chunk_id="test_doc:chunk:1", ), GeneratedQA( - verbatim_span="$12,945,449", # verbatim — must keep + verbatim_span="$14,203,880", # verbatim — must keep question="Net cash from operations in 2023, exact?", - answer="$12,945,449", + answer="$14,203,880", answer_type="numeric", difficulty="easy", evidence_chunk_id="test_doc:chunk:1", @@ -736,7 +736,7 @@ def mock_invoke(prompt: str) -> Any: direct = [r for r in results if r["metadata"]["category"] == "direct_lookup"] assert len(direct) == 1 - assert direct[0]["answer"] == "$12,945,449" + assert direct[0]["answer"] == "$14,203,880" # No needs_review status should leak into the output assert all(r["metadata"].get("validation_status") != "needs_review" for r in results) @@ -956,12 +956,12 @@ def test_entity_audit_rejects_question_containing_corpus_entity() -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. is the principal operating entity incorporated in Nevada.", + content="Contoso Inc. is the principal operating entity incorporated in Nevada.", heading_path=["CORPORATE STRUCTURE"], ), ] - # Question that echoes the corpus entity "Neksai Inc." must fail audit - assert _entity_audit_fails("How is Neksai Inc. structured?", chunks) is True + # Question that echoes the corpus entity "Contoso Inc." must fail audit + assert _entity_audit_fails("How is Contoso Inc. structured?", chunks) is True def test_entity_audit_passes_generic_navigational_question() -> None: @@ -969,7 +969,7 @@ def test_entity_audit_passes_generic_navigational_question() -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. is the principal operating entity.", + content="Contoso Inc. is the principal operating entity.", heading_path=["CORPORATE STRUCTURE"], ), ] @@ -1002,31 +1002,31 @@ def test_entity_audit_is_case_insensitive() -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai BVI Ltd. was incorporated in the British Virgin Islands.", + content="Contoso BVI Ltd. was incorporated in the British Virgin Islands.", heading_path=["CORPORATE STRUCTURE"], ), ] - assert _entity_audit_fails("What is neksai bvi ltd.?", chunks) is True + assert _entity_audit_fails("What is contoso bvi ltd.?", chunks) is True def test_entity_audit_rejects_single_word_entity() -> None: - """First token of a multi-word entity (e.g. 'Neksai' from 'Neksai Inc.') is also audited.""" + """First token of a multi-word entity (e.g. 'Contoso' from 'Contoso Inc.') is also audited.""" chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. is the principal operating entity.", + content="Contoso Inc. is the principal operating entity.", heading_path=["CORPORATE STRUCTURE"], ), ] - # "Neksai" alone — not the full "Neksai Inc." phrase — must still be rejected. - assert _entity_audit_fails("What does Neksai do?", chunks) is True + # "Contoso" alone — not the full "Contoso Inc." phrase — must still be rejected. + assert _entity_audit_fails("What does Contoso do?", chunks) is True def test_entity_audit_scans_extra_text_fields() -> None: """Entity in table_text / pre_text / post_text is also caught.""" chunk = _make_chunk("doc:chunk:1", content="") - chunk["table_text"] = "Neksai BVI Ltd. is a wholly-owned subsidiary." - assert _entity_audit_fails("What does Neksai BVI Ltd. do?", [chunk]) is True + chunk["table_text"] = "Contoso BVI Ltd. is a wholly-owned subsidiary." + assert _entity_audit_fails("What does Contoso BVI Ltd. do?", [chunk]) is True def test_primary_leaf_is_deterministic() -> None: @@ -1058,7 +1058,7 @@ def test_navigational_prompt_omits_chunk_content(mock_settings: MagicMock) -> No chunks = [ _make_chunk( "doc:chunk:1", - content="SECRET_CONTENT: Neksai Inc. revenue was $1M", + content="SECRET_CONTENT: Contoso Inc. revenue was $1M", heading_path=["CORPORATE HISTORY AND STRUCTURE"], ), _make_chunk( @@ -1135,7 +1135,7 @@ def test_navigational_example_schema(mock_settings: MagicMock) -> None: chunks = [ _make_chunk( "doc:chunk:1", - content="The company operates through Neksai Inc. and two BVI subsidiaries.", + content="The company operates through Contoso Inc. and two BVI subsidiaries.", heading_path=["CORPORATE HISTORY AND STRUCTURE"], ), ] @@ -1179,7 +1179,7 @@ def test_navigational_entity_leak_rejected_in_run(mock_settings: MagicMock) -> N chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. operates the principal business.", + content="Contoso Inc. operates the principal business.", heading_path=["CORPORATE STRUCTURE"], ), ] @@ -1188,7 +1188,7 @@ def mock_invoke(prompt: str) -> NavigationalResponse: return NavigationalResponse( questions=[ NavigationalQA( - question="What does Neksai Inc. do?", # entity leak — must be dropped + question="What does Contoso Inc. do?", # entity leak — must be dropped target_section="CORPORATE STRUCTURE", representative_answer="It operates the principal business.", ), @@ -1212,7 +1212,7 @@ def mock_invoke(prompt: str) -> NavigationalResponse: nav = [r for r in results if r["metadata"]["category"] == "navigational"] assert len(nav) == 1 - assert "Neksai" not in nav[0]["question"] + assert "Contoso" not in nav[0]["question"] @patch("tools.dataset_generation.llm.get_settings") @@ -1223,7 +1223,7 @@ def test_navigational_run_tops_up_after_entity_rejections(mock_settings: MagicMo chunks = [ _make_chunk( "doc:chunk:1", - content="Neksai Inc. operates the principal business.", + content="Contoso Inc. operates the principal business.", heading_path=["CORPORATE STRUCTURE"], ), ] @@ -1233,7 +1233,7 @@ def test_navigational_run_tops_up_after_entity_rejections(mock_settings: MagicMo NavigationalResponse( questions=[ NavigationalQA( - question="What does Neksai Inc. do?", + question="What does Contoso Inc. do?", target_section="CORPORATE STRUCTURE", representative_answer="It operates the principal business.", ), diff --git a/tests/unit/test_parser_preprocess.py b/tests/unit/test_parser_preprocess.py index 717210a..92b15d2 100644 --- a/tests/unit/test_parser_preprocess.py +++ b/tests/unit/test_parser_preprocess.py @@ -32,7 +32,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N "avatar_comments_doc": [ { "type": "UncategorizedText", - "text": "NEKSAI INC.", + "text": "CONTOSO INC.", "element_id": "boilerplate-1", "metadata": { "category_depth": 0, @@ -42,7 +42,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N }, { "type": "UncategorizedText", - "text": "NEKSAI INC.", + "text": "CONTOSO INC.", "element_id": "boilerplate-2", "metadata": { "category_depth": 0, @@ -52,7 +52,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N }, { "type": "UncategorizedText", - "text": "NEKSAI INC.", + "text": "CONTOSO INC.", "element_id": "boilerplate-3", "metadata": { "category_depth": 0, @@ -96,7 +96,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N }, { "type": "Table", - "text": "Cash $ 4,431,280 $ 7,351,900", + "text": "Cash $ 3,845,200 $ 6,120,400", "element_id": "table-1", "metadata": { "filename": "avatar-with-comments.docx", @@ -104,7 +104,7 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N "" "" "" - "" + "" "
December 31, 2023December 31, 2023December 31, 2024December 31, 2024
Cash$4,431,280$7,351,900
Cash$3,845,200$6,120,400
" ), }, @@ -138,17 +138,17 @@ def test_run_parser_preprocess_pipeline_normalizes_elements(tmp_path: Path) -> N table_element = next( element for element in normalized_elements if element["element_id"] == "table-1" ) - assert "$4,431,280" in table_element["text"] + assert "$3,845,200" in table_element["text"] assert "December 31, 2023" in table_element["text"] assert "December 31, 2024" in table_element["text"] review_html = Path(result.review_html_path).read_text() rendered_rows = _table_rows_from_html(review_html) - assert "$4,431,280" in review_html - assert "$7,351,900" in review_html + assert "$3,845,200" in review_html + assert "$6,120,400" in review_html assert ">$" not in review_html assert rendered_rows[0] == ["", "December 31, 2023", "", "December 31, 2024"] - assert rendered_rows[1] == ["Cash", "", "$4,431,280", "$7,351,900"] + assert rendered_rows[1] == ["Cash", "", "$3,845,200", "$6,120,400"] warning_codes = {warning.code for warning in result.warnings} assert "missing_page_numbers" in warning_codes @@ -214,14 +214,14 @@ def test_run_parser_preprocess_pipeline_can_enable_table_column_merge(tmp_path: [ { "type": "Table", - "text": "Cash $ 4,431,280 $ 7,351,900", + "text": "Cash $ 3,845,200 $ 6,120,400", "element_id": "table-1", "metadata": { "text_as_html": ( "" "" "" - "" + "" "
December 31, 2023December 31, 2023December 31, 2024December 31, 2024
Cash$4,431,280$7,351,900
Cash$3,845,200$6,120,400
" ), }, @@ -242,7 +242,7 @@ def test_run_parser_preprocess_pipeline_can_enable_table_column_merge(tmp_path: review_html = Path(result.review_html_path).read_text() rendered_rows = _table_rows_from_html(review_html) assert rendered_rows[0] == ["", "December 31, 2023", "December 31, 2024"] - assert rendered_rows[1] == ["Cash", "$4,431,280", "$7,351,900"] + assert rendered_rows[1] == ["Cash", "$3,845,200", "$6,120,400"] def test_run_parser_preprocess_pipeline_describes_images_when_enabled(tmp_path: Path) -> None: @@ -441,14 +441,14 @@ def test_parser_preprocess_command_accepts_config(tmp_path: Path) -> None: [ { "type": "Table", - "text": "Cash $ 4,431,280 $ 7,351,900", + "text": "Cash $ 3,845,200 $ 6,120,400", "element_id": "table-1", "metadata": { "text_as_html": ( "" "" "" - "" + "" "
December 31, 2023December 31, 2023December 31, 2024December 31, 2024
Cash$4,431,280$7,351,900
Cash$3,845,200$6,120,400
" ), }, diff --git a/tests/unit/test_verify_executors.py b/tests/unit/test_verify_executors.py index 3c8d5da..11ee486 100644 --- a/tests/unit/test_verify_executors.py +++ b/tests/unit/test_verify_executors.py @@ -25,7 +25,7 @@ def _write_base(tmp_path: Path) -> Path: "doc_ids": ["f1-doc", "pj-doc"], }, "sources": [ - {"path": "data/raw/ir_document/F1-file.docx", "doc_id": "f1-doc"}, + {"path": "data/raw/ir_document/ir-sample-b.docx", "doc_id": "f1-doc"}, {"path": "data/raw/ir_document/Pj.docx", "doc_id": "pj-doc"}, ], "vector_store": { @@ -62,7 +62,7 @@ def test_derive_config_rewrites_pipeline_and_paths(tmp_path: Path) -> None: assert derived["pipeline"]["artifact_dir"] == str(artifact_dir) # Sources keep their doc_id, paths rewritten under raw_dir by basename. assert derived["sources"] == [ - {"path": str(raw_dir / "F1-file.docx"), "doc_id": "f1-doc"}, + {"path": str(raw_dir / "ir-sample-b.docx"), "doc_id": "f1-doc"}, {"path": str(raw_dir / "Pj.docx"), "doc_id": "pj-doc"}, ] assert derived["vector_store"]["options"]["path"] == str(qdrant_path) diff --git a/tools/dataset_generation/llm.py b/tools/dataset_generation/llm.py index 7af8c7e..d1708ca 100644 --- a/tools/dataset_generation/llm.py +++ b/tools/dataset_generation/llm.py @@ -149,7 +149,7 @@ def _normalize_for_match(text: str) -> str: def _extract_numbers(text: str) -> list[float]: """Pull all numbers from a string (handles commas, decimals, magnitudes).""" out: list[float] = [] - # Find tokens like 12,945,449 or 5.9 (without magnitude) + # Find tokens like 14,203,880 or 5.9 (without magnitude) for m in re.finditer(r"-?[\d,]*\d[\d,]*\.?\d*", text or ""): token = m.group() try: @@ -166,7 +166,7 @@ def _answer_is_grounded( For numeric answers: every number in the answer must appear in the chunk (within `tol` relative tolerance — default 0.5 % to account for cell - formatting differences but reject magnitude rounding like 5.9M vs 5,898,348). + formatting differences but reject magnitude rounding like 5.9M vs 6,357,910). For string answers: the answer's normalized content must be a substring of the normalized chunk. @@ -260,9 +260,9 @@ def _build_corpus_entities(chunks: list[dict[str, Any]]) -> set[str]: """Extract a lowercase entity set from all text fields across chunks. Scans ``content``, ``table_text``, ``pre_text``, and ``post_text``. - Multi-word entity phrases (e.g. ``"neksai inc."``) are stored whole; the + Multi-word entity phrases (e.g. ``"contoso inc."``) are stored whole; the first token is also added when it has ≥5 characters and is not a known legal - suffix so that single-word usage (e.g. ``"Neksai"``) is caught by the audit. + suffix so that single-word usage (e.g. ``"Contoso"``) is caught by the audit. Args: chunks: Chunk dicts from chunks.json. @@ -667,20 +667,20 @@ def _generate_table_lookup( "3. Copy `verbatim_span` unchanged into `answer`.\n\n" "**Strict formatting rules (violations cause the row to be " "rejected):**\n" - '- NO magnitude compression: if the cell shows "$12,945,449" ' - 'the verbatim_span must be "$12,945,449", NOT "$13M" or ' + '- NO magnitude compression: if the cell shows "$14,203,880" ' + 'the verbatim_span must be "$14,203,880", NOT "$13M" or ' '"about $13 million".\n' "- NO unit conversion: copy the number at the scale it appears.\n" '- NO paraphrasing percentages: copy "33.7%" not "~34%".\n' "- Preserve the source's sign convention (parentheses for " "negatives if the table uses them).\n" "- Do NOT append unit words to numbers " - '("$7,351,900" not "$7,351,900 dollars").\n\n' + '("$6,120,400" not "$6,120,400 dollars").\n\n' "**Few-shot examples:**\n\n" "GOOD:\n" - ' verbatim_span: "$4,431,280"\n' + ' verbatim_span: "$3,845,200"\n' ' question: "What was the cash balance as of December 31, 2023?"\n' - ' answer: "$4,431,280"\n' + ' answer: "$3,845,200"\n' " answer_type: numeric\n\n" "GOOD:\n" ' verbatim_span: "33.7%"\n' @@ -689,9 +689,9 @@ def _generate_table_lookup( ' answer: "33.7%"\n' " answer_type: numeric\n\n" "BAD — never do this:\n" - ' verbatim_span: "about $4.4 million" ← compressed; not in ' + ' verbatim_span: "about $3.8 million" ← compressed; not in ' "the table\n" - ' answer: "$4.4M" ← also wrong\n\n' + ' answer: "$3.8M" ← also wrong\n\n' "Other rules:\n" "- Each question must be answerable from exactly ONE table chunk.\n" "- The question MUST name the exact time period from the column " @@ -818,20 +818,20 @@ def _call_direct_lookup(self, chunks: list[dict[str, Any]], count: int) -> list[ "If you cannot find a suitable verbatim span, skip that fact and " "pick a different one.\n\n" "**Strict formatting rules (violations cause rejection):**\n" - '- NO magnitude compression: "$12,945,449" not "$13M" or ' + '- NO magnitude compression: "$14,203,880" not "$13M" or ' '"~$13 million".\n' "- NO paraphrasing: copy names, dates, percentages exactly as " "written in the chunk.\n" - '- NO unit words appended to numbers: "$7,351,900" not ' - '"$7,351,900 dollars".\n' + '- NO unit words appended to numbers: "$6,120,400" not ' + '"$6,120,400 dollars".\n' "- For string answers, use the shortest verbatim span — no " "surrounding narrative.\n\n" "**Few-shot examples:**\n\n" "GOOD:\n" - ' verbatim_span: "$12,945,449"\n' + ' verbatim_span: "$14,203,880"\n' ' question: "What was net cash provided by operating activities' ' for the year ended December 31, 2023?"\n' - ' answer: "$12,945,449"\n' + ' answer: "$14,203,880"\n' " answer_type: numeric\n\n" "GOOD:\n" ' verbatim_span: "AI Avatar Co., Ltd"\n' diff --git a/tools/dataset_generation/template.py b/tools/dataset_generation/template.py index 7b6fc76..d9079e0 100644 --- a/tools/dataset_generation/template.py +++ b/tools/dataset_generation/template.py @@ -216,7 +216,7 @@ def _parse_numeric(text: str) -> float | None: """Parse a financial numeric string into a float. Args: - text: Raw cell value like "$4,431,280" or "(317,662)". + text: Raw cell value like "$3,845,200" or "(352,940)". Returns: Parsed float value, or None if unparseable. diff --git a/xrag/core/generation/openai.py b/xrag/core/generation/openai.py index 54c6e99..0f23d4a 100644 --- a/xrag/core/generation/openai.py +++ b/xrag/core/generation/openai.py @@ -19,7 +19,7 @@ "Numeric extraction rules — when the answer is a number from a table:\n" " 1. Return the EXACT value as it appears in the source, including " "every digit and the original formatting " - '(e.g. "$12,945,449", not "$13M" or "~$13 million"). Do NOT ' + '(e.g. "$14,203,880", not "$13M" or "~$13 million"). Do NOT ' "round, approximate, or compress to a magnitude word.\n" " 2. If multiple cells could match, pick the ONE whose row label " "AND column header EXACTLY match the question's metric name and " @@ -40,8 +40,8 @@ "#3 contains it. When in doubt, attempt to extract.\n" "\n" "When a calculation is required, show the component values from " - "the context first (e.g., 'Revenue 2024 = $45,042,408, Revenue " - "2023 = $23,761,586'), then state the computed result. Never " + "the context first (e.g., 'Revenue 2024 = $48,715,220, Revenue " + "2023 = $26,118,340'), then state the computed result. Never " "invent values not present in the context. Be concise." ) @@ -78,8 +78,8 @@ def _chunks_to_documents(chunks: list[dict[str, Any]]) -> list[Document]: back to markdown. TODO: improve ``src/core/common/converter.py:table_html_to_markdown`` - to drop empty pad columns, merge currency cells (``$ | 4,431,280`` - → ``$4,431,280``), and collapse duplicate multi-row date headers. + to drop empty pad columns, merge currency cells (``$ | 3,845,200`` + → ``$3,845,200``), and collapse duplicate multi-row date headers. Then flip the priority above so markdown wins by default and HTML is the fallback. """