From b30d9c0c71c6b21f3bf0f76ee47f93f17c555291 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 13:50:48 -0400 Subject: [PATCH 01/19] Shortened some ARD titles. --- docs/decisions/0000-stack.md | 2 +- docs/decisions/0001-rag-retrieval-boundary.md | 2 +- .../0002-local-embedding-and-reranking.md | 2 +- docs/decisions/0003-local-single-user-scope.md | 2 +- docs/decisions/0004-cli-batch-ingestion.md | 2 +- docs/decisions/0005-documentation-tooling.md | 16 ++++++++-------- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/decisions/0000-stack.md b/docs/decisions/0000-stack.md index 0ef3186..e535060 100644 --- a/docs/decisions/0000-stack.md +++ b/docs/decisions/0000-stack.md @@ -1,4 +1,4 @@ -# 0. Stack: Python, FastAPI, React, PostgreSQL +# 0. Stack - Status: Accepted - Date: 2026-06-22 diff --git a/docs/decisions/0001-rag-retrieval-boundary.md b/docs/decisions/0001-rag-retrieval-boundary.md index 613e47d..23ced01 100644 --- a/docs/decisions/0001-rag-retrieval-boundary.md +++ b/docs/decisions/0001-rag-retrieval-boundary.md @@ -1,4 +1,4 @@ -# 1. RAG retrieval boundary: a shared `rag_core` library +# 1. RAG Retrieval - Status: Accepted - Date: 2026-06-13 diff --git a/docs/decisions/0002-local-embedding-and-reranking.md b/docs/decisions/0002-local-embedding-and-reranking.md index 21543a2..a3829fd 100644 --- a/docs/decisions/0002-local-embedding-and-reranking.md +++ b/docs/decisions/0002-local-embedding-and-reranking.md @@ -1,4 +1,4 @@ -# 2. Embedding and reranking stack: local open-source models +# 2. Embedding and Reranking - Status: Accepted - Date: 2026-06-13 diff --git a/docs/decisions/0003-local-single-user-scope.md b/docs/decisions/0003-local-single-user-scope.md index d84fa68..319e3ae 100644 --- a/docs/decisions/0003-local-single-user-scope.md +++ b/docs/decisions/0003-local-single-user-scope.md @@ -1,4 +1,4 @@ -# 3. Deployment scope: local single-user +# 3. Deployment - Status: Accepted - Date: 2026-06-13 diff --git a/docs/decisions/0004-cli-batch-ingestion.md b/docs/decisions/0004-cli-batch-ingestion.md index 0eff923..eceb63e 100644 --- a/docs/decisions/0004-cli-batch-ingestion.md +++ b/docs/decisions/0004-cli-batch-ingestion.md @@ -1,4 +1,4 @@ -# 4. Document ingestion: CLI batch +# 4. Ingestion - Status: Accepted - Date: 2026-06-13 diff --git a/docs/decisions/0005-documentation-tooling.md b/docs/decisions/0005-documentation-tooling.md index 786abc0..f07890f 100644 --- a/docs/decisions/0005-documentation-tooling.md +++ b/docs/decisions/0005-documentation-tooling.md @@ -1,4 +1,4 @@ -# 5. Documentation site tooling: Sphinx, MyST, and Furo +# 5. Documentation Site - Status: Accepted - Date: 2026-07-01 @@ -13,13 +13,13 @@ prone to drifting from the code. Both halves — narrative and generated referen static site, deployed by a GitHub Actions workflow like the rest of the project's automation. An initial pass adopted MkDocs + `mkdocstrings` + Material for MkDocs and got as far as a working -`mkdocs.yml`, a generated API reference page, and a deploy workflow. That work surfaced a governance -problem rather than a technical one: MkDocs 2.0 (announced by a maintainer who took over the project -mid-2024) removes the plugin system entirely, has no migration path for existing projects, and is -currently unlicensed. Since `mkdocstrings` — the entire reason to use this stack — is itself a -plugin, pinning `mkdocs<2` only defers the exposure; it does not remove it. For a project being -written from scratch with no legacy MkDocs investment to protect, that is a reason to pick a -foundation without that single point of failure, not to work around it. +`mkdocs.yaml`, a generated API reference page, and a deploy workflow. That work surfaced a +governance problem rather than a technical one: MkDocs 2.0 (announced by a maintainer who took over +the project mid-2024) removes the plugin system entirely, has no migration path for existing +projects, and is currently unlicensed. Since `mkdocstrings` — the entire reason to use this stack — +is itself a plugin, pinning `mkdocs<2` only defers the exposure; it does not remove it. For a +project being written from scratch with no legacy MkDocs investment to protect, that is a reason to +pick a foundation without that single point of failure, not to work around it. Several tool families were considered: From c34bbcfef72c210dc1f966c81be836826af50665 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 13:59:09 -0400 Subject: [PATCH 02/19] Some more title shortening. --- docs/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index d71c7ef..02842a3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,4 +1,4 @@ -# Study Assistant - Architecture & Structure +# Study Assistant: Architecture A personal study assistant over course materials (lecture slides, papers, notes). Answers questions with synthesis from Claude, grounded in retrieved sources, with citations back to the exact source From 1f9f0574e17c7c88c6579253124165d76e2f0cdd Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 14:24:15 -0400 Subject: [PATCH 03/19] Added useful dev commands. --- CHEATSHEET.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 CHEATSHEET.md diff --git a/CHEATSHEET.md b/CHEATSHEET.md new file mode 100644 index 0000000..2aede90 --- /dev/null +++ b/CHEATSHEET.md @@ -0,0 +1,62 @@ +# Dev Cheatsheet + +## Start the DB + +```bash +docker compose up -d +``` + +## Stop / reset DB (after schema changes) + +```bash +docker compose down -v && docker compose up -d +``` + +## Install deps + +```bash +uv sync +``` + +## Tests + +```bash +uv run pytest # everything +uv run pytest -m "not integration" # fast, no DB +uv run pytest -m integration # DB-backed +uv run pytest -k "" # one test +``` + +## Lint / format / typecheck + +```bash +uv run ruff check . +uv run ruff format --check . +uv run mypy +``` + +## Docs preview (clean rebuild + serve) + +```bash +rm -rf site && uv run sphinx-build -E -a -b html docs site -W +uv run python -m http.server -d site 8000 +``` + +## Full pre-PR gate + +```bash +uv run ruff check . && uv run ruff format --check . && uv run mypy && uv run pytest && uv run sphinx-build -b html docs site -W +``` + +## DB connection + +| Field | Value | +| -------- | --------------- | +| Type | PostgreSQL | +| Host | localhost | +| Port | 5432 | +| Database | study_assistant | +| User | study | +| Password | change-me | + +URL: `postgresql://study:change-me@localhost:5432/study_assistant` From 3dc8b889f16bd3882a67c1e3cb23aa632d49a931 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 14:25:59 -0400 Subject: [PATCH 04/19] #2 Deps for rag_core modules. --- packages/rag_core/pyproject.toml | 6 +- pyproject.toml | 10 + uv.lock | 301 +++++++++++++++++++++++++++++++ 3 files changed, 316 insertions(+), 1 deletion(-) diff --git a/packages/rag_core/pyproject.toml b/packages/rag_core/pyproject.toml index 1777be9..c11637a 100644 --- a/packages/rag_core/pyproject.toml +++ b/packages/rag_core/pyproject.toml @@ -4,7 +4,11 @@ version = "0.0.0" description = "Shared retrieval core: ingestion, embedding, hybrid retrieval, reranking, pgvector access." readme = "README.md" requires-python = ">=3.12,<3.13" -dependencies = [] +dependencies = [ + "pgvector>=0.4.2", + "psycopg[binary]>=3.3.4", + "pydantic-settings>=2.14.2", +] [build-system] requires = ["hatchling"] diff --git a/pyproject.toml b/pyproject.toml index 646ceaf..c5a0b61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ dev = [ "myst-parser>=4.0", "sphinx-autodoc-typehints>=2.0", "furo>=2024.8", + "numpy>=2.0", + "sphinx-autobuild>=2025.8.25", ] [tool.ruff] @@ -47,5 +49,13 @@ files = ["packages/rag_core/src", "packages/rag_core/tests"] explicit_package_bases = true namespace_packages = true +# pgvector ships no type stubs / py.typed marker. +[[tool.mypy.overrides]] +module = ["pgvector.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["packages"] +markers = [ + "integration: tests that require a live Postgres+pgvector instance (docker compose up -d)", +] diff --git a/uv.lock b/uv.lock index 93a1353..d54d9d4 100644 --- a/uv.lock +++ b/uv.lock @@ -29,6 +29,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, +] + [[package]] name = "ast-serialize" version = "0.5.0" @@ -109,6 +131,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, ] +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -143,6 +177,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -304,6 +347,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/dc/f3dfb7488b770f3f67e6545085bf2abea5172e88f57b8ad25ef860ca704c/myst_parser-5.1.0-py3-none-any.whl", hash = "sha256:9c91c52b3cdb4d94a6506e4fab4e2f296c7623a0da0dcbe6de1565c3dad67a8a", size = 85817, upload-time = "2026-05-13T09:38:17.904Z" }, ] +[[package]] +name = "numpy" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/05/3d27272d30698dc0ecb7fdfaa41ad70303b444f81722bb99bce1d818638a/numpy-2.5.0.tar.gz", hash = "sha256:5a129578019311b6e56bdd714250f19b518f7dceeeb8d1af5490f4942d3f891c", size = 20652461, upload-time = "2026-06-21T20:57:51.95Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/0a/11486d02add7b1384dff7374d124b1cfbb0ee864dcc9f6a2c0380638cf84/numpy-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:489780423903667933b4ed6197b6ec3b75ea5dd17d1d8f0f38d798feb6921561", size = 16789987, upload-time = "2026-06-21T20:56:16.657Z" }, + { url = "https://files.pythonhosted.org/packages/55/b2/285f48640a181947b4587a3766d21ec1eaa7fea833d4b49957e09da467a2/numpy-2.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ece55976ced6bca95a03ae2839e2e5ccffe8eb6a3e7022415645eb154a81e4e6", size = 11760322, upload-time = "2026-06-21T20:56:19.813Z" }, + { url = "https://files.pythonhosted.org/packages/dd/67/b032db1eb03ca30d16eda3b0c22aaa615338b9263c2fd559d0f29451aca4/numpy-2.5.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c83b664b0e6eee9594fa920cf0639d8af796606d3fad6cc70180c87e4b97c7be", size = 5319605, upload-time = "2026-06-21T20:56:22.173Z" }, + { url = "https://files.pythonhosted.org/packages/b9/83/03fc7300c7c6b6c84c487b1dc80d322817b95fbd1f4dd57a85e23b7198de/numpy-2.5.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bf80333980bf37f523341ddd72c783f39d6829ec7736b9eb99086388a2d52cc2", size = 6653628, upload-time = "2026-06-21T20:56:23.914Z" }, + { url = "https://files.pythonhosted.org/packages/82/49/2ec21730bc63ccfda829323f7040a8ed4715b3852ce658689cf74ee96a8c/numpy-2.5.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1a4874217b36d5ac8fc876f52e39df56f8182c88463e9e2dceabf7ca8b7efb8", size = 15153691, upload-time = "2026-06-21T20:56:25.631Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/f4a3d0637692c49da8ef99d72d52526f92e0a8d6ac4f0ca9f31441b9d9ea/numpy-2.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aaa760137137e8d3c920d27927748215b56014f92667dc9b6c27dfc61249255a", size = 16660066, upload-time = "2026-06-21T20:56:28.009Z" }, + { url = "https://files.pythonhosted.org/packages/3a/2f/c354ec86d1f3f5c19649463b0d39652e160736e5b0a4cd18dff0576715c4/numpy-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7174ce8265fc7f7417d171c9ea8fe905220748893ea67a2a7abe726ec331c4b0", size = 16514638, upload-time = "2026-06-21T20:56:30.26Z" }, + { url = "https://files.pythonhosted.org/packages/06/34/43efdcb319988648580f93c11f1ae82cf7e2faa74925e98e454ae3aa95f8/numpy-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b8c3daaf99de52415d20b42f8e8155c78642cb04207d02f9d317a0dcf1b3fb54", size = 18419647, upload-time = "2026-06-21T20:56:32.41Z" }, + { url = "https://files.pythonhosted.org/packages/71/e2/f5d1676b1d7fb682eb5e9a1641e7ebd2414b3216c370661d1029778908b4/numpy-2.5.0-cp312-cp312-win32.whl", hash = "sha256:6206db0af545d73d068add6d992279145f158428d1da6cc49adc4b630c5d6ee5", size = 6056688, upload-time = "2026-06-21T20:56:34.657Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7c/48f115d1c58a34032facebcd51fdf2d02df2c51d4a46a81dd1197bb2ea6b/numpy-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:6f2d6873e2940c860a309d21e25b1e69af6aaffdd80aa056b04c16380db1c4f2", size = 12419237, upload-time = "2026-06-21T20:56:36.24Z" }, + { url = "https://files.pythonhosted.org/packages/86/26/2e0882f4044d1b1a1b63e875151fb2393389032022a8b7f5657a7996d3b2/numpy-2.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:a55e1eb2bca2cfd17a16b213c99dfc8502d47b0d494224d2122277d0400935ca", size = 10339912, upload-time = "2026-06-21T20:56:38.733Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -322,6 +384,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] +[[package]] +name = "pgvector" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/6c/6d8b4b03b958c02fa8687ec6063c49d952a189f8c91ebbe51e877dfab8f7/pgvector-0.4.2.tar.gz", hash = "sha256:322cac0c1dc5d41c9ecf782bd9991b7966685dee3a00bc873631391ed949513a", size = 31354, upload-time = "2025-12-05T01:07:17.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/26/6cee8a1ce8c43625ec561aff19df07f9776b7525d9002c86bceb3e0ac970/pgvector-0.4.2-py3-none-any.whl", hash = "sha256:549d45f7a18593783d5eec609ea1684a724ba8405c4cb182a0b2b08aeff04e08", size = 27441, upload-time = "2025-12-05T01:07:16.536Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -331,6 +405,101 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, + { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, + { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, + { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -356,6 +525,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -378,6 +556,18 @@ wheels = [ name = "rag-core" version = "0.0.0" source = { editable = "packages/rag_core" } +dependencies = [ + { name = "pgvector" }, + { name = "psycopg", extra = ["binary"] }, + { name = "pydantic-settings" }, +] + +[package.metadata] +requires-dist = [ + { name = "pgvector", specifier = ">=0.4.2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.3.4" }, + { name = "pydantic-settings", specifier = ">=2.14.2" }, +] [[package]] name = "requests" @@ -474,6 +664,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, ] +[[package]] +name = "sphinx-autobuild" +version = "2025.8.25" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "sphinx" }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/20/56411b52f917696995f5ad27d2ea7e9492c84a043c5b49a3a3173573cd93/sphinx_autobuild-2025.8.25-py3-none-any.whl", hash = "sha256:b750ac7d5a18603e4665294323fd20f6dcc0a984117026d1986704fa68f0379a", size = 12535, upload-time = "2025-08-25T18:44:54.164Z" }, +] + [[package]] name = "sphinx-autodoc-typehints" version = "3.12.0" @@ -552,6 +759,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, ] +[[package]] +name = "starlette" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, +] + [[package]] name = "study-assistant" version = "0.0.0" @@ -562,10 +782,12 @@ dev = [ { name = "furo" }, { name = "mypy" }, { name = "myst-parser" }, + { name = "numpy" }, { name = "pytest" }, { name = "rag-core" }, { name = "ruff" }, { name = "sphinx" }, + { name = "sphinx-autobuild" }, { name = "sphinx-autodoc-typehints" }, ] @@ -576,10 +798,12 @@ dev = [ { name = "furo", specifier = ">=2024.8" }, { name = "mypy", specifier = ">=1.11" }, { name = "myst-parser", specifier = ">=4.0" }, + { name = "numpy", specifier = ">=2.0" }, { name = "pytest", specifier = ">=8.0" }, { name = "rag-core", editable = "packages/rag_core" }, { name = "ruff", specifier = ">=0.6" }, { name = "sphinx", specifier = ">=8.0" }, + { name = "sphinx-autobuild", specifier = ">=2025.8.25" }, { name = "sphinx-autodoc-typehints", specifier = ">=2.0" }, ] @@ -592,6 +816,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" @@ -600,3 +845,59 @@ sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] + +[[package]] +name = "uvicorn" +version = "0.49.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", size = 91284, upload-time = "2026-06-03T22:01:30.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", size = 71376, upload-time = "2026-06-03T22:01:29.037Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, + { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, + { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, + { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, + { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, + { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, + { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, + { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, +] + +[[package]] +name = "websockets" +version = "16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365, upload-time = "2026-01-10T09:22:46.787Z" }, + { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038, upload-time = "2026-01-10T09:22:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328, upload-time = "2026-01-10T09:22:49.809Z" }, + { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915, upload-time = "2026-01-10T09:22:51.071Z" }, + { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152, upload-time = "2026-01-10T09:22:52.224Z" }, + { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583, upload-time = "2026-01-10T09:22:53.443Z" }, + { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880, upload-time = "2026-01-10T09:22:55.033Z" }, + { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261, upload-time = "2026-01-10T09:22:56.251Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693, upload-time = "2026-01-10T09:22:57.478Z" }, + { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, +] From fc5fb2f2a6b93c2d05b14dc27fdf0b129ad1e8ca Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 14:30:17 -0400 Subject: [PATCH 05/19] #2 SQL schema for documents and chunks. Also adds new indexes (HNSW and GIN) for pgvector support. --- .../rag_core/src/rag_core/store/schema.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 packages/rag_core/src/rag_core/store/schema.py diff --git a/packages/rag_core/src/rag_core/store/schema.py b/packages/rag_core/src/rag_core/store/schema.py new file mode 100644 index 0000000..979ef6c --- /dev/null +++ b/packages/rag_core/src/rag_core/store/schema.py @@ -0,0 +1,49 @@ +"""SQL schema for documents and chunks with pgvector support. + +Schema evolution is handled by ``Store.initialize_schema`` running this idempotent +``CREATE ... IF NOT EXISTS`` bootstrap. A versioned migration mechanism is deferred until the +first breaking change actually needs one — most likely the re-embed / ``vector(N)`` dimension +change tracked by issue #15. Until then there is no existing data to migrate. +""" + +from rag_core.config import settings + +# Extension and Tables +SCHEMA_SQL = f""" +-- Ensure vector extension exists +CREATE EXTENSION IF NOT EXISTS vector; + +-- Documents table: Tracks source files +CREATE TABLE IF NOT EXISTS documents ( + id UUID PRIMARY KEY, + filename TEXT NOT NULL, + -- UNIQUE makes re-ingesting the same file idempotent (upsert target below) and + -- doubles as the lookup index for get_document_by_hash. Single-user scope keys on + -- file_hash alone; multi-user would migrate this to (user_id, file_hash). + file_hash TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Chunks table: Stores semantic segments and embeddings +CREATE TABLE IF NOT EXISTS chunks ( + id UUID PRIMARY KEY, + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + content TEXT NOT NULL, + metadata JSONB NOT NULL DEFAULT '{{}}', + embedding vector({settings.embedding_dimension}) NOT NULL, + tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED, + user_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- HNSW index for vector similarity search (cosine distance) +CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON chunks +USING hnsw (embedding vector_cosine_ops); + +-- GIN index for full-text search (BM25) +CREATE INDEX IF NOT EXISTS idx_chunks_tsv ON chunks USING gin(tsv); + +-- Index on document_id for fast cleanup/retrieval +CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id); +""" From ee64b04547653ab10e159b163c179c11f8eecd2a Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 15:49:37 -0400 Subject: [PATCH 06/19] #2 DB config and client impl: CRUD documents/chunks. --- packages/rag_core/src/rag_core/config.py | 27 +++++ .../rag_core/src/rag_core/store/__init__.py | 3 + .../rag_core/src/rag_core/store/client.py | 109 ++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 packages/rag_core/src/rag_core/config.py create mode 100644 packages/rag_core/src/rag_core/store/__init__.py create mode 100644 packages/rag_core/src/rag_core/store/client.py diff --git a/packages/rag_core/src/rag_core/config.py b/packages/rag_core/src/rag_core/config.py new file mode 100644 index 0000000..0bb56c1 --- /dev/null +++ b/packages/rag_core/src/rag_core/config.py @@ -0,0 +1,27 @@ +from uuid import UUID + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Configuration for rag_core. + + Loads from environment variables or .env file. + """ + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + # Database connection string. Required — supplied via env or .env (see .env.example); + # no default so a missing DATABASE_URL fails loudly instead of silently using a fake one. + database_url: str + + # Vector dimensions (pinned to bge-m3) + embedding_dimension: int = 1024 + + # Default user for single-user local scope (ADR 0003) + default_user_id: UUID = UUID("00000000-0000-0000-0000-000000000000") + + +# Required fields are populated from the environment / .env by pydantic-settings at runtime, +# which mypy's dataclass-style view of the model can't see — hence the call-arg ignore. +settings = Settings() # type: ignore[call-arg] diff --git a/packages/rag_core/src/rag_core/store/__init__.py b/packages/rag_core/src/rag_core/store/__init__.py new file mode 100644 index 0000000..831037f --- /dev/null +++ b/packages/rag_core/src/rag_core/store/__init__.py @@ -0,0 +1,3 @@ +from rag_core.store.client import Store + +__all__ = ["Store"] diff --git a/packages/rag_core/src/rag_core/store/client.py b/packages/rag_core/src/rag_core/store/client.py new file mode 100644 index 0000000..0e40223 --- /dev/null +++ b/packages/rag_core/src/rag_core/store/client.py @@ -0,0 +1,109 @@ +from typing import Any +from uuid import UUID + +import psycopg +from pgvector.psycopg import register_vector +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb + +from rag_core.config import settings +from rag_core.store.schema import SCHEMA_SQL + + +class Store: + """Store client for pgvector-backed RAG storage.""" + + def __init__(self, database_url: str | None = None) -> None: + self.database_url = database_url or settings.database_url + + def initialize_schema(self) -> None: + """Idempotently creates extension and tables.""" + with psycopg.connect(self.database_url, autocommit=True) as conn: + conn.execute(SCHEMA_SQL) + + def get_connection(self) -> psycopg.Connection[Any]: + """Returns a connection with vector support registered.""" + conn = psycopg.connect(self.database_url, row_factory=dict_row) + register_vector(conn) + return conn + + def add_document( + self, + id: UUID, + filename: str, + file_hash: str, + user_id: UUID | None = None, + ) -> UUID: + """Insert document metadata idempotently, returning the stored document id. + + On re-ingest of the same ``file_hash`` the existing row is kept (its id is + returned and the filename refreshed), so callers can re-run ingestion safely. + """ + uid = user_id or settings.default_user_id + with self.get_connection() as conn: + cur = conn.execute( + """ + INSERT INTO documents (id, filename, file_hash, user_id) + VALUES (%s, %s, %s, %s) + ON CONFLICT (file_hash) DO UPDATE SET filename = EXCLUDED.filename + RETURNING id + """, + (id, filename, file_hash, uid), + ) + row = cur.fetchone() + assert row is not None # RETURNING on upsert always yields a row + stored_id: UUID = row["id"] + return stored_id + + def update_document(self, id: UUID, filename: str) -> None: + """Update mutable document metadata (currently just the filename).""" + with self.get_connection() as conn: + conn.execute( + "UPDATE documents SET filename = %s WHERE id = %s", + (filename, id), + ) + + def delete_document(self, id: UUID) -> None: + """Delete a document and its chunks (chunks cascade on the FK).""" + with self.get_connection() as conn: + conn.execute("DELETE FROM documents WHERE id = %s", (id,)) + + def get_document_by_hash(self, file_hash: str) -> dict[str, Any] | None: + """Find a document by its file hash.""" + with self.get_connection() as conn: + cur = conn.execute( + "SELECT id, filename, file_hash, user_id FROM documents WHERE file_hash = %s", + (file_hash,), + ) + return cur.fetchone() + + # TODO: Consider handling partial success in batch inserts. + def add_chunks( + self, + chunks: list[dict[str, Any]], + user_id: UUID | None = None, + ) -> None: + """Batch insert chunks. + + Expected chunk dict: {id, document_id, content, metadata, embedding} + """ + uid = user_id or settings.default_user_id + with self.get_connection() as conn: + with conn.cursor() as cur: + copy_sql = ( + "COPY chunks (id, document_id, content, metadata, embedding, user_id) " + "FROM STDIN WITH (FORMAT BINARY)" + ) + with cur.copy(copy_sql) as copy: + copy.set_types(["uuid", "uuid", "text", "jsonb", "vector", "uuid"]) + for chunk in chunks: + copy.write_row( + ( + chunk["id"], + chunk["document_id"], + chunk["content"], + Jsonb(chunk["metadata"]), + chunk["embedding"], + uid, + ) + ) From 467e0b79cf4d8781d5e0c68c519f76acb8866356 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 17:27:33 -0400 Subject: [PATCH 07/19] #2 Store tests against a test DB, also included in CI. --- .env.example | 2 + .github/workflows/ci.yaml | 17 ++++ infra/postgres/init.sql | 7 ++ packages/rag_core/src/rag_core/config.py | 9 +- .../rag_core/src/rag_core/store/client.py | 8 +- packages/rag_core/tests/conftest.py | 47 ++++++++++ packages/rag_core/tests/test_store.py | 86 +++++++++++++++++++ 7 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 packages/rag_core/tests/conftest.py create mode 100644 packages/rag_core/tests/test_store.py diff --git a/.env.example b/.env.example index ed6babd..949d0b8 100644 --- a/.env.example +++ b/.env.example @@ -5,11 +5,13 @@ POSTGRES_USER=study POSTGRES_PASSWORD=change-me POSTGRES_DB=study_assistant +POSTGRES_TEST_DB=study_assistant_test POSTGRES_HOST=localhost POSTGRES_PORT=5432 # Connection string used by the app/services (host networking from the host machine). DATABASE_URL=postgresql://study:change-me@localhost:5432/study_assistant +TEST_DATABASE_URL=postgresql://study:change-me@localhost:5432/study_assistant_test # ---- Generation (used in later phases) ---- # Claude is the one hosted dependency; ingestion + retrieval stay on-machine. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 36a6cc2..9cff983 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,21 @@ jobs: checks: name: Lint, typecheck, test runs-on: ubuntu-latest + # Integration tests (rag_core/store) run against a real pgvector instance. + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: study + POSTGRES_PASSWORD: change-me + POSTGRES_DB: study_assistant_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U study -d study_assistant_test" + --health-interval 5s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v5 @@ -33,6 +48,8 @@ jobs: - name: Pytest run: uv run pytest + env: + TEST_DATABASE_URL: postgresql://study:change-me@localhost:5432/study_assistant_test docs: name: Docs build diff --git a/infra/postgres/init.sql b/infra/postgres/init.sql index 7e217fb..89fbcb4 100644 --- a/infra/postgres/init.sql +++ b/infra/postgres/init.sql @@ -2,3 +2,10 @@ -- Phase 0 only makes the pgvector extension available; tables and the vector(N) -- columns are added in Phase 1 (rag_core/store). CREATE EXTENSION IF NOT EXISTS vector; + +-- Second database on the same instance, used only by integration tests so their table-wide +-- teardown never touches real ingested data. POSTGRES_DB auto-creates only one database, so the +-- test database must be created explicitly here. Keep this name in sync with POSTGRES_TEST_DB / +-- TEST_DATABASE_URL in .env. The pgvector extension is created inside it by +-- Store.initialize_schema() when the tests first connect. +CREATE DATABASE study_assistant_test; diff --git a/packages/rag_core/src/rag_core/config.py b/packages/rag_core/src/rag_core/config.py index 0bb56c1..d640e3c 100644 --- a/packages/rag_core/src/rag_core/config.py +++ b/packages/rag_core/src/rag_core/config.py @@ -11,9 +11,8 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") - # Database connection string. Required — supplied via env or .env (see .env.example); - # no default so a missing DATABASE_URL fails loudly instead of silently using a fake one. - database_url: str + # Optional so components (docs, tests, etc.) can build without a real database. + database_url: str | None = None # Vector dimensions (pinned to bge-m3) embedding_dimension: int = 1024 @@ -22,6 +21,4 @@ class Settings(BaseSettings): default_user_id: UUID = UUID("00000000-0000-0000-0000-000000000000") -# Required fields are populated from the environment / .env by pydantic-settings at runtime, -# which mypy's dataclass-style view of the model can't see — hence the call-arg ignore. -settings = Settings() # type: ignore[call-arg] +settings = Settings() diff --git a/packages/rag_core/src/rag_core/store/client.py b/packages/rag_core/src/rag_core/store/client.py index 0e40223..629429d 100644 --- a/packages/rag_core/src/rag_core/store/client.py +++ b/packages/rag_core/src/rag_core/store/client.py @@ -14,7 +14,13 @@ class Store: """Store client for pgvector-backed RAG storage.""" def __init__(self, database_url: str | None = None) -> None: - self.database_url = database_url or settings.database_url + url = database_url or settings.database_url + if url is None: + raise RuntimeError( + "No database URL configured. Pass Store(database_url=...) or set DATABASE_URL " + "in the environment / .env (see .env.example)." + ) + self.database_url = url def initialize_schema(self) -> None: """Idempotently creates extension and tables.""" diff --git a/packages/rag_core/tests/conftest.py b/packages/rag_core/tests/conftest.py new file mode 100644 index 0000000..389baf1 --- /dev/null +++ b/packages/rag_core/tests/conftest.py @@ -0,0 +1,47 @@ +"""Shared pytest fixtures and test-only configuration for the rag_core test suite. + +Anything reusable across test modules lives here: pytest auto-loads `conftest.py`, so test files use +these fixtures without importing them. Keep test-only settings in `IntegrationConfig` (never in the +app's `rag_core.config.Settings`), and add more test config / fixtures here as the suite grows. +""" + +from collections.abc import Iterator + +import pytest +from pydantic_settings import BaseSettings, SettingsConfigDict + +from rag_core.store import Store + + +class IntegrationConfig(BaseSettings): + """Test-only settings, separate from the app's `rag_core.config.Settings`. + + Reads `TEST_DATABASE_URL` from the environment or `.env` so integration tests connect to an + isolated database, never the app's real `DATABASE_URL`. Add further test-only settings here. + """ + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + test_database_url: str | None = None + + +@pytest.fixture(scope="session") +def integration_config() -> IntegrationConfig: + return IntegrationConfig() + + +@pytest.fixture +def store(integration_config: IntegrationConfig) -> Iterator[Store]: + # Integration tests must never touch the app's real DATABASE_URL + if integration_config.test_database_url is None: + raise RuntimeError( + "TEST_DATABASE_URL is not set. Integration tests require an isolated test database " + "(never the app's DATABASE_URL). Copy .env.example to .env; the test database is " + "provisioned automatically by infra/postgres/init.sql on `docker compose up -d`." + ) + s = Store(database_url=integration_config.test_database_url) + s.initialize_schema() + yield s + # Clears them together without needing CASCADE for the chunks -> documents foreign key. + with s.get_connection() as conn: + conn.execute("TRUNCATE documents, chunks") diff --git a/packages/rag_core/tests/test_store.py b/packages/rag_core/tests/test_store.py new file mode 100644 index 0000000..9033754 --- /dev/null +++ b/packages/rag_core/tests/test_store.py @@ -0,0 +1,86 @@ +import uuid + +import numpy as np +import pytest + +from rag_core.config import settings +from rag_core.store import Store + +pytestmark = pytest.mark.integration + + +def test_initialize_schema(store: Store) -> None: + # Idempotent: running again must not error. + store.initialize_schema() + + +def test_document_and_chunks(store: Store) -> None: + doc_id = uuid.uuid4() + file_hash = "test-hash-" + str(uuid.uuid4()) + + stored_id = store.add_document(id=doc_id, filename="test.md", file_hash=file_hash) + assert stored_id == doc_id + + doc = store.get_document_by_hash(file_hash) + assert doc is not None + assert doc["id"] == doc_id + + chunk_id = uuid.uuid4() + embedding = np.random.rand(settings.embedding_dimension).tolist() + store.add_chunks( + [ + { + "id": chunk_id, + "document_id": doc_id, + "content": "This is a test chunk.", + "metadata": {"page": 1}, + "embedding": embedding, + } + ] + ) + + with store.get_connection() as conn: + cur = conn.execute("SELECT content, embedding FROM chunks WHERE id = %s", (chunk_id,)) + row = cur.fetchone() + assert row is not None + assert row["content"] == "This is a test chunk." + # Vector column round-trips: register_vector returns a numpy array. + assert row["embedding"].shape == (settings.embedding_dimension,) + assert np.allclose(row["embedding"], embedding) + + +def test_add_document_is_idempotent(store: Store) -> None: + file_hash = "test-hash-" + str(uuid.uuid4()) + + first = store.add_document(id=uuid.uuid4(), filename="a.md", file_hash=file_hash) + # Same hash, different id + filename: the original row is kept, id is returned. + second = store.add_document(id=uuid.uuid4(), filename="b.md", file_hash=file_hash) + assert second == first + + doc = store.get_document_by_hash(file_hash) + assert doc is not None + assert doc["id"] == first + assert doc["filename"] == "b.md" + + +def test_delete_document_cascades_chunks(store: Store) -> None: + doc_id = uuid.uuid4() + chunk_id = uuid.uuid4() + store.add_document(id=doc_id, filename="d.md", file_hash="test-hash-" + str(uuid.uuid4())) + store.add_chunks( + [ + { + "id": chunk_id, + "document_id": doc_id, + "content": "chunk", + "metadata": {}, + "embedding": np.random.rand(settings.embedding_dimension).tolist(), + } + ] + ) + + store.delete_document(doc_id) + + with store.get_connection() as conn: + assert conn.execute("SELECT 1 FROM documents WHERE id = %s", (doc_id,)).fetchone() is None + assert conn.execute("SELECT 1 FROM chunks WHERE id = %s", (chunk_id,)).fetchone() is None From 3f4e066ef752c00865d31be1ae49606c984bbdc2 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Thu, 2 Jul 2026 17:28:18 -0400 Subject: [PATCH 08/19] #2 Store documentation update. --- AGENTS.md | 7 ++++++- README.md | 7 ++++--- docs/index.md | 16 ++++++++-------- docs/reference/index.md | 32 +++++++++++++++++++++----------- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 74f3960..67d0647 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,8 @@ Anthropic API. - Read [`docs/architecture.md`](docs/architecture.md), it is the full design. - Read ADRs under [`docs/decisions/`](docs/decisions/), locked decisions live there. -- Read the [GitHub issues](https://github.com/sid-ak/study_assistant/issues), work is tracked as phases there. +- Read the [GitHub issues](https://github.com/sid-ak/study_assistant/issues), work is tracked as + phases there. - Read [`README.md`](README.md), Status blurb states intent. - Compare against repo, which is the ground truth. - Scan with `ls` and compare with directory structure in `architecture.md` to gauge progress. @@ -52,6 +53,10 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. - Integration tests are marked `@pytest.mark.integration`. Scope a run with `uv run pytest -m "not integration"` (fast, no DB) or `-m integration` (DB-backed); target one test with `uv run pytest -k ""`. +- Integration tests run against a separate database (`TEST_DATABASE_URL`, e.g. + `study_assistant_test`), never the app's `DATABASE_URL`, and truncate their tables between tests. + `docker compose up -d` remains the only setup step — the test database is created automatically by + `infra/postgres/init.sql` on first init, so a fresh clone needs nothing extra. - Lint, format, and typecheck must also be green: `uv run ruff check . && uv run ruff format --check . && uv run mypy`. Fix every error and type failure until the whole suite is green before you merge. diff --git a/README.md b/README.md index 87024f1..c7b0bfe 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ free; only answer generation calls the Anthropic API. ## Status -Early scaffolding (Phase 0 — Foundations). The repo currently contains the `uv` workspace root, the -shared `packages/rag_core/` library, and the Postgres + pgvector dev stack. Later phases add the -CLI, MCP server, FastAPI backend, and React frontend. +Early scaffolding (Phase 1 — Storage + Schema). The repo contains the `uv` workspace root, the +shared `packages/rag_core/` library with the pgvector storage layer, and the Postgres + pgvector dev +stack. Later phases add ingestion, retrieval logic, the CLI, MCP server, FastAPI backend, and React +frontend. ## Prerequisites diff --git a/docs/index.md b/docs/index.md index 81088b0..32f7ca6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,6 +11,14 @@ architecture.md ``` +```{toctree} +:hidden: +:maxdepth: 2 +:caption: API + +reference/index.md +``` + ```{toctree} :hidden: :maxdepth: 1 @@ -24,11 +32,3 @@ decisions/0003-local-single-user-scope.md decisions/0004-cli-batch-ingestion.md decisions/0005-documentation-tooling.md ``` - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: API Reference - -reference/index.md -``` diff --git a/docs/reference/index.md b/docs/reference/index.md index 30d7f7d..7b81db3 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -1,9 +1,5 @@ # API Reference -Generated from docstrings and the `mypy --strict` type hints by Sphinx autodoc (see -[ADR 0005](../decisions/0005-documentation-tooling.md)). One directive block per module, so the -reference stays navigable rather than one giant page. - ## rag_core ```{eval-rst} @@ -13,12 +9,26 @@ reference stays navigable rather than one giant page. :show-inheritance: ``` -## Modules added as phases land +## rag_core.config + +```{eval-rst} +.. automodule:: rag_core.config + :members: + :show-inheritance: +``` + +## rag_core.store.client + +```{eval-rst} +.. automodule:: rag_core.store.client + :members: + :show-inheritance: +``` -The following modules are not yet in the tree; add one `automodule` block each (identical in shape -to the `rag_core` block above) as they land: +## rag_core.store.schema -- `rag_core.config` — settings -- `rag_core.store.client` — pgvector access -- `rag_core.store.schema` — table/index DDL -- later phases: `rag_core.ingest`, `rag_core.embed`, `rag_core.rerank`, `rag_core.retrieve` +```{eval-rst} +.. automodule:: rag_core.store.schema + :members: + :show-inheritance: +``` From 081f8fed38512bf1741d6554146a5e73f228fd07 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Fri, 3 Jul 2026 19:15:48 -0400 Subject: [PATCH 09/19] #2 Reflected architectural changes in docs: interfaces, embedder flexibility and generation model agnostic. --- docs/architecture.md | 62 ++++++------------- docs/assets/architecture.svg | 10 +-- docs/decisions/0001-rag-retrieval-boundary.md | 2 +- .../0002-local-embedding-and-reranking.md | 4 +- .../0006-interface-first-architecture.md | 56 +++++++++++++++++ docs/decisions/0007-generation-backend.md | 54 ++++++++++++++++ .../0008-per-model-embedding-columns.md | 57 +++++++++++++++++ docs/decisions/README.md | 19 +++--- docs/index.md | 3 + 9 files changed, 207 insertions(+), 60 deletions(-) create mode 100644 docs/decisions/0006-interface-first-architecture.md create mode 100644 docs/decisions/0007-generation-backend.md create mode 100644 docs/decisions/0008-per-model-embedding-columns.md diff --git a/docs/architecture.md b/docs/architecture.md index 02842a3..472ce07 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,9 +1,9 @@ # Study Assistant: Architecture A personal study assistant over course materials (lecture slides, papers, notes). Answers questions -with synthesis from Claude, grounded in retrieved sources, with citations back to the exact source -slide/page. The locked decisions that shaped all of this are recorded as Architectural Decision -Records (ADRs) in [`decisions/`](decisions/). +with synthesis grounded in retrieved sources, with citations back to the exact source slide/page. +The decisions that shaped all of this are recorded as Architectural Decision Records (ADRs) in +[`decisions/`](decisions/). ## Outline @@ -21,14 +21,6 @@ everything on machine, offline, and for free. Embeddings and the cross-encoder r there are no per-call API costs and no network dependency for retrieval and because nothing transits to a vendor, the corpus stays private. -> NOTE -> "Offline and for free" scopes to ingestion and retrieval, which run entirely on-machine. -> Generation is the one exception: it calls Claude (`claude-opus-4-8`) through the Anthropic API, a -> hosted, paid, online dependency. The model-agnostic goal therefore applies _below_ the reasoning -> layer — the embedder, reranker, and coding agent are swappable, while the reasoning model is -> deliberately Claude-coupled for synthesis quality. A path to closing that gap is in -> [Future Scope](#future-scope). - Domain-level goals that shaped the architecture also include keeping a human in the loop (HITL) to steer an ambiguous question before a slow rerank runs over the wrong material, keeping the reasoning model swappable rather than foundational, and keeping the reasoning layer ignorant of where chunks @@ -39,6 +31,13 @@ live or how they're scored so retrieval stays a clean, replaceable concern. The system is a retrieval-augmented pipeline organized into three lanes: Ingestion, Retrieval, and Generation over a single PostgreSQL/pgvector store, with a React frontend on top. +One theme runs through the whole system: each swappable component is defined as a small interface +with a concrete implementation behind it. Callers depend only on the abstraction, so an +implementation is chosen by config and replaced with a fake in tests. The embedder, reranker, +generator, and store access are all built this way, which is what keeps the system swappable and +testable across many implementations. See +[ADR 0006](decisions/0006-interface-first-architecture.md). + ![RAG pipeline architecture](./assets/architecture.svg) ### Ingestion @@ -62,8 +61,10 @@ against regressions and demonstrates that reranking measurably improves results. Generation runs in the FastAPI backend, where a LangGraph state machine orchestrates the agent loop — reaching retrieval through the MCP server's tools, pausing at HITL checkpoints, and synthesizing a -grounded answer with Claude (`claude-opus-4-8`, adaptive thinking, streaming). Responses stream to -the React frontend over SSE, with citations resolving back to the exact source slide or page. +grounded answer. Responses stream to the React frontend over SSE, with citations resolving back to +the exact source slide or page. + +See [ADR 0007](decisions/0007-generation-backend.md) for the backend and candidate models. ## Governance and Conventions @@ -87,22 +88,6 @@ coder or coding agent in the future. > each scoped file exactly when an agent works in that directory) triggers on `CLAUDE.md`. Keeping > the stub preserves all of that while the canonical content stays under `AGENTS.md`. -### Primary Conventions - -- Retrieval lives only in `rag_core`. MCP server and API are consumers, never reimplementers. -- Lazy model loading: no `torch` import at module top-level outside `embed/` and `rerank/`. -- Embedder/reranker behind a small interface so the rest of the system is agnostic to the concrete - model; the only real lock-in is the pgvector embedding dimension (`vector(N)`). - -### Other Conventions - -- Models: Claude via the Anthropic SDK, `claude-opus-4-8`, adaptive thinking, streaming. -- Python tooling: `uv` workspace; lint/format/typecheck in pre-commit and CI. -- The `study` CLI is its own top-level package (`cli/`), depending on `rag_core` as a workspace - dependency. Keeps the ingestion entry point cleanly separated from the library. -- Model weights are cached in a named Docker volume, not baked into images — the multi-GB `bge` - download happens once. - ## Directory ```text @@ -171,28 +156,17 @@ study_assistant/ └── postgres/ # pgvector init.sql, model-weights volume notes ``` -> NOTE -> The Python packages are documented on a Sphinx + MyST + Furo site (`docs/conf.py`, deployed by -> `.github/workflows/docs-deploy.yaml`; see [ADR 0005](decisions/0005-documentation-tooling.md)). - ## Future Scope -### Model Agnostic - -A hybrid local/Claude generation approach would extend the model-agnostic goal through the reasoning -layer and make a fully free, offline run possible end-to-end. The generation node would sit behind a -small interface — the same pattern already used for the embedder and reranker — so config selects -the model per run: a local model served through an OpenAI-compatible runtime (Ollama, llama.cpp, or -vLLM) as the free default, with Claude reserved for harder questions where synthesis quality matters -most. - ### Cloud Deployable A cloud-deployable path is the natural next step beyond local: it would add managed Postgres, a registry push, and prod-vs-local config. The heavier lift there is the local `bge` models, which are expensive to host in the cloud — so a future cloud move may also revisit the embedding/reranking -stack (e.g. a hosted pairing like Voyage's `voyage-3` + `rerank-2.5`), a change gated by the -pgvector `vector(N)` dimension lock-in and therefore a re-embed plus schema migration. +stack (e.g. a hosted pairing like Voyage's `voyage-3` + `rerank-2.5`). With per-model embedding +columns ([ADR 0008](decisions/0008-per-model-embedding-columns.md)) that swap is a re-embed into a +new `vector(N)` column and index rather than a destructive rebuild of the shared one — additive and +reversible, though pgvector still requires a fixed dimension per column. ### Multi User Auth diff --git a/docs/assets/architecture.svg b/docs/assets/architecture.svg index 4688d9d..69005aa 100644 --- a/docs/assets/architecture.svg +++ b/docs/assets/architecture.svg @@ -1,6 +1,6 @@ RAG pipeline architecture diagram - Three-lane architecture showing ingestion, retrieval, and generation layers connecting materials to a React frontend via Claude. + Three-lane architecture showing ingestion, retrieval, and generation layers connecting materials to a React frontend via the Generator. @@ -135,11 +135,11 @@ - + - Claude - opus-4-8, streaming + Generator + streaming @@ -150,7 +150,7 @@ SSE stream - + diff --git a/docs/decisions/0001-rag-retrieval-boundary.md b/docs/decisions/0001-rag-retrieval-boundary.md index 23ced01..4bdfcbb 100644 --- a/docs/decisions/0001-rag-retrieval-boundary.md +++ b/docs/decisions/0001-rag-retrieval-boundary.md @@ -6,7 +6,7 @@ ## Context A core goal of the project is to decouple tooling from the model — the agent reaches retrieval -through a custom MCP server rather than importing it directly. Claude handles reasoning and +through a custom MCP server rather than importing it directly. The model handles reasoning and synthesis; retrieval (chunking, embedding, hybrid search, reranking) is a separate concern that several components need: the MCP server (to expose it as tools), the FastAPI backend (for ingestion/admin paths), and the CLI (for batch ingestion). diff --git a/docs/decisions/0002-local-embedding-and-reranking.md b/docs/decisions/0002-local-embedding-and-reranking.md index a3829fd..b1df9cc 100644 --- a/docs/decisions/0002-local-embedding-and-reranking.md +++ b/docs/decisions/0002-local-embedding-and-reranking.md @@ -5,8 +5,8 @@ ## Context -Claude has no embeddings endpoint, so the retrieval stack is an independent vendor/tech choice with -two distinct stages: +The generation model provides no embeddings endpoint, so the retrieval stack is an independent +vendor/tech choice with two distinct stages: - Embedding — turns each chunk and each query into a vector stored in pgvector; powers the dense half of hybrid search. diff --git a/docs/decisions/0006-interface-first-architecture.md b/docs/decisions/0006-interface-first-architecture.md new file mode 100644 index 0000000..5be3efb --- /dev/null +++ b/docs/decisions/0006-interface-first-architecture.md @@ -0,0 +1,56 @@ +# 6. Interface-First Architecture + +- Status: Accepted +- Date: 2026-07-02 + +## Context + +Several components in the system must be replaceable: the embedder and reranker (open-weight models +that will change as the field moves), the generation backend (which must carry no vendor tie-in — +see [ADR 0007](0007-generation-backend.md)), and pgvector-backed store access. The system also has +to be testable without standing up multi-GB models or a live database on every run — CI and fast +unit tests need substitutes for the heavy, external pieces. + +[ADR 0001](0001-rag-retrieval-boundary.md) already decided where retrieval physically lives (a +shared `rag_core` library) and noted, per component, that the embedder and reranker sit behind "a +small interface." This ADR promotes that from a per-component remark to an explicit, project-wide +principle, so new components inherit the shape by default rather than by coincidence. + +The alternative — wiring concrete classes (a specific embedder, a specific model client, direct +driver calls) straight into their callers — is quicker to write first but couples the whole system +to those choices: swapping a model means editing every call site, and testing means loading the real +model or hitting the real database. + +## Decision + +Build every swappable component as a small interface with concrete implementations behind it. This +is the general theme of the architecture, not a one-off for any single lane. + +- Interfaces are Python `Protocol`s — structural typing, so an implementation conforms by shape with + no base class to inherit. +- Callers depend on the `Protocol`, never a concrete type. +- The concrete implementation is selected by config and injected. + +It applies across the codebase: + +- Embedder and reranker — behind interfaces so the concrete `bge` models are swappable (see + [ADR 0002](0002-local-embedding-and-reranking.md)). +- Generator — behind a `Generator` interface with a vendor-neutral adapter (see + [ADR 0007](0007-generation-backend.md)). +- Store — pgvector access behind an interface, so retrieval logic does not depend on raw SQL or + driver calls. + +## Consequences + +- Swappability: an implementation changes by config, not by editing call sites — the point of the + exercise, and what lets models and backends move as the field does. +- Testability: tests substitute lightweight fakes for a `Protocol`, so unit tests run without + loading multi-GB models or a live database, while integration tests exercise the real + implementations. +- `Protocol`/structural typing keeps this lightweight — no inheritance hierarchy — and `mypy` + (strict) checks conformance at the boundary. +- The one hard lock-in that no interface can hide is the pgvector `vector(N)` embedding dimension, + baked into the schema (see [ADR 0002](0002-local-embedding-and-reranking.md)); everything else + stays behind an abstraction. +- A small indirection cost, and the standing discipline that callers depend on the `Protocol` rather + than reaching around it to a concrete class. This is carried as a convention in `AGENTS.md`. diff --git a/docs/decisions/0007-generation-backend.md b/docs/decisions/0007-generation-backend.md new file mode 100644 index 0000000..594d810 --- /dev/null +++ b/docs/decisions/0007-generation-backend.md @@ -0,0 +1,54 @@ +# 7. Generation Backend + +- Status: Accepted +- Date: 2026-07-02 + +## Context + +Generation is the reasoning lane: a LangGraph loop reaches retrieval through the MCP server's tools, +pauses at HITL checkpoints, and synthesizes a grounded, cited answer. Per the interface-first +architecture ([ADR 0006](0006-interface-first-architecture.md)), the reasoning model sits behind a +`Generator` interface; this ADR records which backend goes behind that interface, and why. + +Two things drive the choice. First, removing the project's only vendor tie-in — the reasoning model +was its one online, paid, hosted dependency. Second, keeping the reasoning model on-machine, which +closes the last gap to a fully offline, free run: ingestion and retrieval already run locally, so a +local generation backend makes the whole pipeline private and free end-to-end, with no prompts or +chunks transiting to a vendor. + +Unlike the embedder, the generation backend carries no schema lock-in: the embedding dimension is +fixed in the pgvector `vector(N)` column (see [ADR 0002](0002-local-embedding-and-reranking.md)), +but a generation model can be swapped per run with no migration. So the concrete model should not be +pinned the way the embedder is. + +## Decision + +The `Generator` has one adapter, deliberately vendor-neutral: it speaks the OpenAI-compatible +chat-completions protocol — a de-facto standard implemented by local runtimes (Ollama, vLLM, +llama.cpp) and many hosted providers alike, not a dependency on OpenAI or any single vendor. No +vendor-specific SDK enters the generation lane. By default the adapter points at a local runtime for +a fully offline, free run; it can also point at any hosted endpoint that speaks the same protocol. + +Candidate open-weight local backends, as a dated snapshot (2026-07) — a snapshot, not a locked +choice, and to be re-verified before implementation: + +- Qwen3.6 — primary candidate. The most reliable tool/function-calling among open-weight models, + long context, runs via Ollama with MLX acceleration on Apple Silicon (Ollama 0.19+). Best fit for + the LangGraph MCP tool-calling loop and HITL checkpoints. +- IBM Granite 4 (Apache 2.0, 3B–32B) — secondary/conservative option. Purpose-trained for function + calling with a smaller footprint, but less capable at open-ended reasoning. +- Kimi K2.6, DeepSeek V4 Pro, GLM-5.1 — also strong on agentic benchmarks at this time. Larger, and + worth reconsidering if hardware allows. + +## Consequences + +- No vendor tie-in: generation depends only on the OpenAI-compatible protocol, so the project has no + hosted, paid, online dependency and the whole pipeline can run offline and free end-to-end. +- The concrete model is chosen in config per run and is not locked. The list above is a 2026-07 + research snapshot that must be re-checked against the current landscape before implementation, not + treated as a fixed decision. +- Open-weight models vary in tool-calling reliability, so model selection is weighted toward + function-calling quality, and the golden-set eval discipline used for retrieval extends naturally + to checking generation behavior. +- Adopting the OpenAI-compatible protocol rather than a richer vendor SDK trades some + provider-specific features for portability — the intended trade, since portability is the point. diff --git a/docs/decisions/0008-per-model-embedding-columns.md b/docs/decisions/0008-per-model-embedding-columns.md new file mode 100644 index 0000000..e7bfc42 --- /dev/null +++ b/docs/decisions/0008-per-model-embedding-columns.md @@ -0,0 +1,57 @@ +# 8. Per-Model Embedding Columns + +- Status: Accepted +- Date: 2026-07-02 + +## Context + +[ADR 0002](0002-local-embedding-and-reranking.md) pins `bge-m3` and notes that its output dimension +is baked into the pgvector `vector(N)` column and its index, so changing embedders means a re-embed +plus a schema migration — the "main lock-in." Today `chunks` has a single `embedding vector(1024)` +column with one HNSW index, and the dimension is read from the global `settings.embedding_dimension` +at import time inside `store/schema.py`. + +Two problems follow from that shape: + +- The dimension is a global config value, read at import to build the schema SQL, so the store and + its schema are coupled to a global settings read rather than told the dimension explicitly. +- One shared column means switching embedders is a destructive, in-place `ALTER` of the single + column that every row and every query depends on. Old and new embeddings cannot coexist, so the + swap is all-or-nothing and blocking: the table is mid-migration until every chunk is re-embedded. + +The [interface-first work](0006-interface-first-architecture.md) makes the embedder swappable in +code; this ADR addresses the schema side so a swap is not destructive in storage. + +## Decision + +Two changes, together making an embedder swap additive rather than in-place. + +- Dimension as an explicit parameter. Move the embedding dimension off the global + `settings.embedding_dimension` and pass it explicitly when an embedder is registered/initialized. + This decouples the schema and store from a global settings read. It is a hygiene fix — on its own + it does not remove the migration cost of switching embedders, it only removes the global coupling. +- Per-model embedding columns. Each active embedder gets its own `vector(N)` column and its own + index, tagged by a model identifier — e.g. `embedding_bge_m3 vector(1024)` with a dedicated HNSW + index — plus a recorded marker of which model is "current." Adding an embedder becomes an additive + schema change (a new column and a new index), not an in-place `ALTER` of a shared column. Old and + new embeddings coexist during a migration, and the old column is dropped only once the new one is + fully backfilled and validated. + +## Consequences + +- Embedder swaps become additive and reversible instead of destructive and blocking: backfill the + new column alongside the old, validate with the golden-set eval (ADR 0002), flip the "current" + marker, then drop the old column. A failed or in-progress migration leaves the existing column + intact and queryable. +- This does not eliminate the underlying constraint that pgvector — and every ANN index, HNSW + included — requires a fixed dimension per column and per index at creation time. That is + structural to vector indexes generally, not specific to this project. What changes is scope: the + fixed- dimension constraint stops applying globally to the whole table and becomes local to one + column, so a swap is an additive column rather than a table-wide rebuild. +- Cost while two embedders are active: extra columns and indexes, so more storage and more write + work during a migration. Ingestion writes and retrieval reads must target the current model's + column, selected by the "current" marker. +- This refines the "main lock-in" framing in ADR 0002 and ADR 0006: the dimension is still fixed per + column, but it no longer gates the whole table. The future embedding-stack change contemplated in + Future Scope (e.g. a hosted Voyage pairing on a cloud move) is a re-embed into a new column, not a + blocking rebuild of the shared one. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index b9937a9..39a1fdf 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -7,11 +7,14 @@ way it is. The architecture and structure live in [`../architecture.md`](../arch phased build plan is tracked as [GitHub issues](https://github.com/sid-ak/study_assistant/issues?q=is%3Aissue%20label%3Aphase). -| ADR | Decision | Status | -| --------------------------------------------- | ------------------------------------------------------- | -------- | -| [0000](0000-stack.md) | Stack: Python, FastAPI, PostgreSQL, React | Accepted | -| [0001](0001-rag-retrieval-boundary.md) | RAG retrieval boundary: a shared `rag_core` library | Accepted | -| [0002](0002-local-embedding-and-reranking.md) | Embedding and reranking stack: local open-source models | Accepted | -| [0003](0003-local-single-user-scope.md) | Deployment scope: local single-user | Accepted | -| [0004](0004-cli-batch-ingestion.md) | Document ingestion: CLI batch | Accepted | -| [0005](0005-documentation-tooling.md) | Documentation site tooling: Sphinx, MyST, and Furo | Accepted | +| ADR | Decision | Status | +| --------------------------------------------- | ----------------------------------------------------------- | -------- | +| [0000](0000-stack.md) | Stack: Python, FastAPI, PostgreSQL, React | Accepted | +| [0001](0001-rag-retrieval-boundary.md) | RAG retrieval boundary: a shared `rag_core` library | Accepted | +| [0002](0002-local-embedding-and-reranking.md) | Embedding and reranking stack: local open-source models | Accepted | +| [0003](0003-local-single-user-scope.md) | Deployment scope: local single-user | Accepted | +| [0004](0004-cli-batch-ingestion.md) | Document ingestion: CLI batch | Accepted | +| [0005](0005-documentation-tooling.md) | Documentation site tooling: Sphinx, MyST, and Furo | Accepted | +| [0006](0006-interface-first-architecture.md) | Interface-first architecture: `Protocol`s + implementations | Accepted | +| [0007](0007-generation-backend.md) | Generation backend: vendor-neutral, OpenAI-compatible | Accepted | +| [0008](0008-per-model-embedding-columns.md) | Per-model embedding columns (additive embedder swaps) | Accepted | diff --git a/docs/index.md b/docs/index.md index 32f7ca6..d3633bb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,4 +31,7 @@ decisions/0002-local-embedding-and-reranking.md decisions/0003-local-single-user-scope.md decisions/0004-cli-batch-ingestion.md decisions/0005-documentation-tooling.md +decisions/0006-interface-first-architecture.md +decisions/0007-generation-backend.md +decisions/0008-per-model-embedding-columns.md ``` From d54200afd0661862ce96a418b8613592f7bc943f Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Fri, 3 Jul 2026 23:30:47 -0400 Subject: [PATCH 10/19] Added TDD as a rule to root AGENTS.md to enforce moving forward. --- AGENTS.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 67d0647..2ee5af5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,10 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. ## Testing instructions +- Strict TDD, in two passes (binding, not a preference). For any new feature or issue, pass one + writes tests only — fully describing the expected behavior — and adds no implementation code; run + them and expect red. Pass two adds implementation until they pass (green). This applies to both + unit and integration tests, per the `@pytest.mark.integration` marker in `pyproject.toml`. - The CI plan is in `.github/workflows/ci.yaml`: a `checks` job (ruff lint, ruff format check, mypy, pytest against a pgvector service) and a `docs` job that builds the Sphinx site (`uv run sphinx-build -b html docs site -W`, build-only — deployment lives in `docs-deploy.yaml`). @@ -62,7 +66,9 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. failure until the whole suite is green before you merge. - Docs must build clean: `uv run sphinx-build -b html docs site -W` (CI runs this too — `-W` turns Sphinx warnings into errors, catching broken toctrees, anchors, and autodoc import failures). -- Add or update tests for the code you change, even if nobody asked. +- Tests come first, not alongside. Writing or updating tests for the code you change is mandatory + (even if nobody asked), and under the two-pass rule above it happens in pass one — before the + implementation exists — not in the same pass as the code. - Schema changes need a fresh DB: `initialize_schema()` is `CREATE ... IF NOT EXISTS` and will not retrofit constraints, so run `docker compose down -v && docker compose up -d` after editing `store/schema.py`. From c1c35f5a6b9f2038e9b6eea61ae4565e2483810e Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Sat, 4 Jul 2026 00:05:53 -0400 Subject: [PATCH 11/19] Trimmed rag_core AGENTS.md file and recorded the three pass process in the root AGENTS.md. --- AGENTS.md | 16 +++++--- packages/rag_core/AGENTS.md | 75 ++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2ee5af5..a543647 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,10 +45,14 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. ## Testing instructions -- Strict TDD, in two passes (binding, not a preference). For any new feature or issue, pass one - writes tests only — fully describing the expected behavior — and adds no implementation code; run - them and expect red. Pass two adds implementation until they pass (green). This applies to both - unit and integration tests, per the `@pytest.mark.integration` marker in `pyproject.toml`. +- Strict TDD, in three passes (binding, not a preference). For any new feature or issue: pass one + writes tests only — fully describing the expected behavior — with no interface or implementation + code, run and expect red. Pass two defines the interface(s) the code will satisfy — the `Protocol` + types and signatures, per the interface-first architecture + ([ADR 0006](docs/decisions/0006-interface-first-architecture.md)) — with no implementation logic. + Pass three adds the implementation behind those interfaces until the tests pass (green). This + applies to both unit and integration tests, per the `@pytest.mark.integration` marker in + `pyproject.toml`. - The CI plan is in `.github/workflows/ci.yaml`: a `checks` job (ruff lint, ruff format check, mypy, pytest against a pgvector service) and a `docs` job that builds the Sphinx site (`uv run sphinx-build -b html docs site -W`, build-only — deployment lives in `docs-deploy.yaml`). @@ -67,8 +71,8 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. - Docs must build clean: `uv run sphinx-build -b html docs site -W` (CI runs this too — `-W` turns Sphinx warnings into errors, catching broken toctrees, anchors, and autodoc import failures). - Tests come first, not alongside. Writing or updating tests for the code you change is mandatory - (even if nobody asked), and under the two-pass rule above it happens in pass one — before the - implementation exists — not in the same pass as the code. + (even if nobody asked), and under the three-pass rule above it happens in pass one — before the + interface or implementation exists — not in the same pass as the code. - Schema changes need a fresh DB: `initialize_schema()` is `CREATE ... IF NOT EXISTS` and will not retrofit constraints, so run `docker compose down -v && docker compose up -d` after editing `store/schema.py`. diff --git a/packages/rag_core/AGENTS.md b/packages/rag_core/AGENTS.md index ac2a566..f6c5b4c 100644 --- a/packages/rag_core/AGENTS.md +++ b/packages/rag_core/AGENTS.md @@ -1,40 +1,39 @@ # rag_core — Agent Context -`rag_core` is the shared retrieval library: the single, authoritative implementation of chunking, -embedding, hybrid retrieval, reranking, and pgvector access. See the root -[`AGENTS.md`](../../AGENTS.md) for project-wide context. - -## Golden rule - -Retrieval lives only here. The MCP server, the API, and the CLI are _consumers_ of `rag_core`, never -reimplementers. If retrieval logic is being written anywhere else, it belongs here instead -([ADR 0001](../../docs/decisions/0001-rag-retrieval-boundary.md)). The public surface is an internal -API — treat it as versioned and change it deliberately. - -## Conventions specific to this package - -- Lazy model loading. No `torch` import at module top-level outside `embed/` and `rerank/`. - Importing `rag_core` (or any non-model module) must not pull in PyTorch or load weights - ([ADR 0002](../../docs/decisions/0002-local-embedding-and-reranking.md)). -- Embedder and reranker sit behind a small interface so the rest of the system is agnostic to the - concrete model. The one hard lock-in is the pgvector embedding dimension (`vector(N)`): query and - document vectors must come from the same model, pinned in config, not chosen per call. Changing - embedders later means a re-embed plus schema migration. -- `user_id` seam. Schema carries a defaulted `user_id` column on relevant tables so multi-user auth - can be layered on later without reshaping the data model - ([ADR 0003](../../docs/decisions/0003-local-single-user-scope.md)). - -## Layout (built out across phases) - -```text -src/rag_core/ -├── ingest/ # pptx/pdf/md parsing, semantic chunking (Phase 2/3) -├── embed/ # bge-m3 embedder, lazy torch load (Phase 4) -├── rerank/ # bge-reranker-v2-m3 cross-encoder (Phase 5) -├── retrieve/ # dense + BM25 + RRF fusion (Phase 5) -├── store/ # pgvector access, schema, migrations (Phase 1) -└── config.py # pinned models, dimensions, settings -``` - -Only the package skeleton exists today. Modules are added in the phases noted above; until then, -keep new code out of this package unless its phase has started. +`rag_core` is the shared retrieval library. See the root [`AGENTS.md`](../../AGENTS.md) for +project-wide context and [`docs/architecture.md`](../../docs/architecture.md) for where this package +sits and its module layout. + +Its rules are binding decisions recorded as ADRs, not restated here: read the relevant record in +[`docs/decisions/`](../../docs/decisions/) before changing what it governs. The ones that bind +`rag_core` are the retrieval boundary (0001), the embedding/reranking stack and `vector(N)` +dimension (0002), the single-user `user_id` seam (0003), interface-first `Protocol`s (0006), and +per-model embedding columns (0008). + +## Known drift from ADRs + +Two accepted ADRs are ahead of the code — don't assume the shape below already matches the ADR just +because it's "Accepted": + +- **ADR 0006 (interface-first):** `store/__init__.py` exports the concrete `Store` class directly; + there is no `Protocol` in front of it yet. Store access is still concrete everywhere it's called. + Introducing the `Protocol` (plus a fake for unit tests) is open work, not done. +- **ADR 0008 (per-model embedding columns):** `config.py` still reads a single global + `settings.embedding_dimension`, and `schema.py` still builds one shared `embedding vector(N)` + column — the pre-0008 shape ADR 0008 exists to replace. Explicit per-call dimensions and + per-model columns aren't implemented; tracked in + [issue #15](https://github.com/sid-ak/study_assistant/issues/15), which is still open. + +## Store conventions + +Not ADR-level, but load-bearing for anyone adding to `store/`: + +- Every write method takes an optional `user_id: UUID | None`, defaulting to + `settings.default_user_id` (the ADR 0003 seam). New methods should follow the same signature + rather than hardcoding the default user or dropping the parameter. +- `get_connection()` calls `register_vector(conn)` so `vector` columns round-trip as numpy arrays. + Any code that opens its own `psycopg.connect(...)` outside `Store` must call `register_vector` + itself or vector columns won't (de)serialize. +- `add_chunks` bulk-inserts via the `COPY ... FORMAT BINARY` protocol, not per-row `INSERT` — keep + new batch-write paths on `COPY` for the same throughput reason. It's currently all-or-nothing per + batch (see the TODO in `client.py`); don't assume partial-failure handling exists. From 5c3f0168acb0af389ec08e4737458a035162e05e Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Sat, 4 Jul 2026 17:53:57 -0400 Subject: [PATCH 12/19] #2 Defined a StoreProtocol to ADR 0006, backed by pg DB store and an in memory store. --- .../rag_core/src/rag_core/store/__init__.py | 4 +- .../rag_core/src/rag_core/store/client.py | 10 +++ .../rag_core/src/rag_core/store/memory.py | 83 +++++++++++++++++++ .../rag_core/src/rag_core/store/protocol.py | 57 +++++++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 packages/rag_core/src/rag_core/store/memory.py create mode 100644 packages/rag_core/src/rag_core/store/protocol.py diff --git a/packages/rag_core/src/rag_core/store/__init__.py b/packages/rag_core/src/rag_core/store/__init__.py index 831037f..83ed2b2 100644 --- a/packages/rag_core/src/rag_core/store/__init__.py +++ b/packages/rag_core/src/rag_core/store/__init__.py @@ -1,3 +1,5 @@ from rag_core.store.client import Store +from rag_core.store.memory import InMemoryStore +from rag_core.store.protocol import StoreProtocol -__all__ = ["Store"] +__all__ = ["InMemoryStore", "Store", "StoreProtocol"] diff --git a/packages/rag_core/src/rag_core/store/client.py b/packages/rag_core/src/rag_core/store/client.py index 629429d..14ca0c0 100644 --- a/packages/rag_core/src/rag_core/store/client.py +++ b/packages/rag_core/src/rag_core/store/client.py @@ -113,3 +113,13 @@ def add_chunks( uid, ) ) + + def get_chunks_by_document(self, document_id: UUID) -> list[dict[str, Any]]: + """Return every chunk row belonging to ``document_id``, or ``[]`` if none exist.""" + with self.get_connection() as conn: + cur = conn.execute( + "SELECT id, document_id, content, metadata, embedding, user_id " + "FROM chunks WHERE document_id = %s", + (document_id,), + ) + return cur.fetchall() diff --git a/packages/rag_core/src/rag_core/store/memory.py b/packages/rag_core/src/rag_core/store/memory.py new file mode 100644 index 0000000..b6a541f --- /dev/null +++ b/packages/rag_core/src/rag_core/store/memory.py @@ -0,0 +1,83 @@ +"""In-memory ``StoreProtocol`` implementation for unit tests. + +A lightweight fake so tests that only need storage semantics — not real pgvector — run without a +database. It mirrors the concrete ``Store``'s CRUD contract: hash-keyed idempotent document upsert, +FK-style cascade on delete, and the same ``user_id`` default. It does not model vector or full-text +search; those are exercised by the DB-backed integration tests against ``Store``. +""" + +from typing import Any +from uuid import UUID + +from rag_core.config import settings + + +class InMemoryStore: + """Dict-backed fake conforming to ``StoreProtocol``.""" + + def __init__(self) -> None: + # Keyed by document / chunk id. ``document`` rows mirror Store's SELECT shape. + self.documents: dict[UUID, dict[str, Any]] = {} + self.chunks: dict[UUID, dict[str, Any]] = {} + + def initialize_schema(self) -> None: + """No-op: an in-memory store has no schema to create.""" + + def add_document( + self, + id: UUID, + filename: str, + file_hash: str, + user_id: UUID | None = None, + ) -> UUID: + uid = user_id or settings.default_user_id + # Idempotent by file_hash: keep the original row/id, refresh the filename. + for doc in self.documents.values(): + if doc["file_hash"] == file_hash: + doc["filename"] = filename + stored_id: UUID = doc["id"] + return stored_id + self.documents[id] = { + "id": id, + "filename": filename, + "file_hash": file_hash, + "user_id": uid, + } + return id + + def update_document(self, id: UUID, filename: str) -> None: + doc = self.documents.get(id) + if doc is not None: + doc["filename"] = filename + + def delete_document(self, id: UUID) -> None: + self.documents.pop(id, None) + # Cascade: drop chunks belonging to the document. + self.chunks = { + cid: chunk for cid, chunk in self.chunks.items() if chunk["document_id"] != id + } + + def get_document_by_hash(self, file_hash: str) -> dict[str, Any] | None: + for doc in self.documents.values(): + if doc["file_hash"] == file_hash: + return dict(doc) + return None + + def add_chunks( + self, + chunks: list[dict[str, Any]], + user_id: UUID | None = None, + ) -> None: + uid = user_id or settings.default_user_id + for chunk in chunks: + self.chunks[chunk["id"]] = { + "id": chunk["id"], + "document_id": chunk["document_id"], + "content": chunk["content"], + "metadata": chunk["metadata"], + "embedding": chunk["embedding"], + "user_id": uid, + } + + def get_chunks_by_document(self, document_id: UUID) -> list[dict[str, Any]]: + return [dict(c) for c in self.chunks.values() if c["document_id"] == document_id] diff --git a/packages/rag_core/src/rag_core/store/protocol.py b/packages/rag_core/src/rag_core/store/protocol.py new file mode 100644 index 0000000..1a292c7 --- /dev/null +++ b/packages/rag_core/src/rag_core/store/protocol.py @@ -0,0 +1,57 @@ +"""The store interface. + +``StoreProtocol`` is the structural contract retrieval and ingestion code depends on, so callers +never bind to the concrete ``Store`` (pgvector) or to raw driver calls. ``Store`` conforms by shape, +and ``InMemoryStore`` is a lightweight fake satisfying the same contract for unit tests with no live +database. + +``get_connection`` is intentionally absent: it returns a raw ``psycopg`` connection, a driver +dependency retrieval logic should stay clear of. Integration tests use it on the concrete ``Store`` +directly. +""" + +from typing import Any, Protocol, runtime_checkable +from uuid import UUID + + +@runtime_checkable +class StoreProtocol(Protocol): + """Structural interface for pgvector-backed document/chunk storage.""" + + def initialize_schema(self) -> None: + """Idempotently create the extension and tables.""" + ... + + def add_document( + self, + id: UUID, + filename: str, + file_hash: str, + user_id: UUID | None = None, + ) -> UUID: + """Insert document metadata idempotently, returning the stored document id.""" + ... + + def update_document(self, id: UUID, filename: str) -> None: + """Update mutable document metadata (currently just the filename).""" + ... + + def delete_document(self, id: UUID) -> None: + """Delete a document and its chunks (chunks cascade).""" + ... + + def get_document_by_hash(self, file_hash: str) -> dict[str, Any] | None: + """Find a document by its file hash, or ``None`` if absent.""" + ... + + def add_chunks( + self, + chunks: list[dict[str, Any]], + user_id: UUID | None = None, + ) -> None: + """Batch insert chunks: ``{id, document_id, content, metadata, embedding}``.""" + ... + + def get_chunks_by_document(self, document_id: UUID) -> list[dict[str, Any]]: + """Return every chunk belonging to ``document_id``, or ``[]`` if none exist.""" + ... From 544b180f060afc99e5683d44addaa361f4b9d8fe Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Sat, 4 Jul 2026 18:46:00 -0400 Subject: [PATCH 13/19] #2 Added parameterized tests for StoreProtocol and an implementation specific one for db store. --- conftest.py | 16 +++ packages/rag_core/tests/conftest.py | 27 ++++- packages/rag_core/tests/test_db_store.py | 59 ++++++++++ packages/rag_core/tests/test_store.py | 131 +++++++++++++++-------- pyproject.toml | 1 + 5 files changed, 188 insertions(+), 46 deletions(-) create mode 100644 conftest.py create mode 100644 packages/rag_core/tests/test_db_store.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..7c51698 --- /dev/null +++ b/conftest.py @@ -0,0 +1,16 @@ +"""Root pytest configuration shared across every workspace package. + +Auto-applies the ``unit`` marker to any test not marked ``integration``, so ``uv run pytest -m +unit`` selects the fast, DB-free suite without each test file opting in, and keeps doing so as tests +are added. Integration tests opt in explicitly with ``@pytest.mark.integration`` (or a module-level +``pytestmark``); everything else is unit by default. +""" + +import pytest + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Tag every collected test lacking the ``integration`` mark with the ``unit`` mark.""" + for item in items: + if item.get_closest_marker("integration") is None: + item.add_marker(pytest.mark.unit) diff --git a/packages/rag_core/tests/conftest.py b/packages/rag_core/tests/conftest.py index 389baf1..7af337a 100644 --- a/packages/rag_core/tests/conftest.py +++ b/packages/rag_core/tests/conftest.py @@ -10,7 +10,7 @@ import pytest from pydantic_settings import BaseSettings, SettingsConfigDict -from rag_core.store import Store +from rag_core.store import InMemoryStore, Store, StoreProtocol class IntegrationConfig(BaseSettings): @@ -27,11 +27,17 @@ class IntegrationConfig(BaseSettings): @pytest.fixture(scope="session") def integration_config() -> IntegrationConfig: + """Session-scoped test-only settings, read once from the environment / `.env`.""" return IntegrationConfig() @pytest.fixture -def store(integration_config: IntegrationConfig) -> Iterator[Store]: +def db_store(integration_config: IntegrationConfig) -> Iterator[Store]: + """Real, DB-backed `Store` against `TEST_DATABASE_URL`, tables truncated after each test. + + Behavior that only the concrete `Store` has (raw vector round-tripping, + schema DDL). Shared `StoreProtocol` behavior uses the parametrized `store` fixture instead. + """ # Integration tests must never touch the app's real DATABASE_URL if integration_config.test_database_url is None: raise RuntimeError( @@ -45,3 +51,20 @@ def store(integration_config: IntegrationConfig) -> Iterator[Store]: # Clears them together without needing CASCADE for the chunks -> documents foreign key. with s.get_connection() as conn: conn.execute("TRUNCATE documents, chunks") + + +@pytest.fixture( + params=[ + "memory", + pytest.param("db", marks=pytest.mark.integration), + ] +) +def store(request: pytest.FixtureRequest) -> StoreProtocol: + """Every ``StoreProtocol`` implementation, once each. + + Backs the contract suite. + """ + if request.param == "memory": + return InMemoryStore() + db_store: Store = request.getfixturevalue("db_store") + return db_store diff --git a/packages/rag_core/tests/test_db_store.py b/packages/rag_core/tests/test_db_store.py new file mode 100644 index 0000000..3af51a6 --- /dev/null +++ b/packages/rag_core/tests/test_db_store.py @@ -0,0 +1,59 @@ +"""Integration tests for behavior that only exists on the concrete, DB-backed ``Store``. + +Shared CRUD behavior common to every ``StoreProtocol`` implementation lives in +`test_store.py` instead, parametrized against both `Store` and `InMemoryStore`. What stays +here is genuinely Store-only: raw vector round-tripping through pgvector and idempotent schema DDL, +neither of which is expressible through `StoreProtocol` (`get_connection()` is deliberately absent +from it — see `rag_core/AGENTS.md`). +""" + +import uuid + +import numpy as np +import pytest + +from rag_core.config import settings +from rag_core.store import Store + +pytestmark = pytest.mark.integration + + +def test_initialize_schema(db_store: Store) -> None: + """initialize_schema is idempotent — a second run against an existing schema must not error.""" + db_store.initialize_schema() + + +def test_document_and_chunks_round_trip_vector(db_store: Store) -> None: + """A chunk's embedding round-trips through pgvector and reads back as a numpy array.""" + doc_id = uuid.uuid4() + file_hash = "test-hash-" + str(uuid.uuid4()) + content = "This is a test chunk." + + stored_id = db_store.add_document(id=doc_id, filename="test.md", file_hash=file_hash) + assert stored_id == doc_id + + doc = db_store.get_document_by_hash(file_hash) + assert doc is not None + assert doc["id"] == doc_id + + chunk_id = uuid.uuid4() + embedding = np.random.rand(settings.embedding_dimension).tolist() + db_store.add_chunks( + [ + { + "id": chunk_id, + "document_id": doc_id, + "content": content, + "metadata": {"page": 1}, + "embedding": embedding, + } + ] + ) + + with db_store.get_connection() as conn: + cur = conn.execute("SELECT content, embedding FROM chunks WHERE id = %s", (chunk_id,)) + row = cur.fetchone() + assert row is not None + assert row["content"] == content + assert row["embedding"].shape == (settings.embedding_dimension,) + assert np.allclose(row["embedding"], embedding) diff --git a/packages/rag_core/tests/test_store.py b/packages/rag_core/tests/test_store.py index 9033754..3e0f0b1 100644 --- a/packages/rag_core/tests/test_store.py +++ b/packages/rag_core/tests/test_store.py @@ -1,72 +1,106 @@ -import uuid +"""Shared behavior contract for the store seam. + +Every test here takes `store: StoreProtocol` (the parametrized fixture in `conftest.py`) and +runs once against `InMemoryStore` and once against the real, DB-backed `Store` — labeled +`[memory]` / `[db]` in test output. This is deliberate: `add_document`, `add_chunks`, and the rest +of `StoreProtocol` must behave identically regardless of which implementation is behind it, and a +single shared assertion can't silently drift out of sync the way two hand-duplicated copies could. -import numpy as np -import pytest +Only `StoreProtocol` methods are used here — nothing peeks at implementation internals (no +`get_connection()`, no reaching into `InMemoryStore.chunks`). Behavior that genuinely can't be +expressed through the Protocol (raw vector round-tripping, schema DDL idempotency) stays in +`test_db_store.py` against the concrete `Store`. +""" + +import uuid from rag_core.config import settings -from rag_core.store import Store +from rag_core.store import InMemoryStore, Store, StoreProtocol + + +def test_add_document_returns_stored_id(store: StoreProtocol) -> None: + """A fresh insert returns the id it was given.""" + doc_id = uuid.uuid4() + stored_id = store.add_document(id=doc_id, filename="a.md", file_hash="h1") + assert stored_id == doc_id -pytestmark = pytest.mark.integration + +def test_add_document_is_idempotent_by_hash(store: StoreProtocol) -> None: + """Re-adding the same file_hash keeps the original id and refreshes the filename.""" + first = store.add_document(id=uuid.uuid4(), filename="a.md", file_hash="dup") + second = store.add_document(id=uuid.uuid4(), filename="b.md", file_hash="dup") + assert second == first + + doc = store.get_document_by_hash("dup") + assert doc is not None + assert doc["id"] == first + assert doc["filename"] == "b.md" -def test_initialize_schema(store: Store) -> None: - # Idempotent: running again must not error. - store.initialize_schema() +def test_get_document_by_hash_missing_returns_none(store: StoreProtocol) -> None: + """An unknown hash yields None, not an error.""" + assert store.get_document_by_hash("nope") is None -def test_document_and_chunks(store: Store) -> None: +def test_update_document_changes_filename(store: StoreProtocol) -> None: + """update_document overwrites the stored filename.""" doc_id = uuid.uuid4() - file_hash = "test-hash-" + str(uuid.uuid4()) + store.add_document(id=doc_id, filename="old.md", file_hash="h") + store.update_document(id=doc_id, filename="new.md") + doc = store.get_document_by_hash("h") + assert doc is not None + assert doc["filename"] == "new.md" - stored_id = store.add_document(id=doc_id, filename="test.md", file_hash=file_hash) - assert stored_id == doc_id - doc = store.get_document_by_hash(file_hash) +def test_user_id_defaults_to_settings_default(store: StoreProtocol) -> None: + """Omitting user_id falls back to settings.default_user_id (the ADR 0003 seam).""" + doc_id = uuid.uuid4() + store.add_document(id=doc_id, filename="a.md", file_hash="h") + doc = store.get_document_by_hash("h") + assert doc is not None + assert doc["user_id"] == settings.default_user_id + + +def test_user_id_override_is_honored(store: StoreProtocol) -> None: + """An explicit user_id is stored instead of the default.""" + doc_id = uuid.uuid4() + uid = uuid.uuid4() + store.add_document(id=doc_id, filename="a.md", file_hash="h", user_id=uid) + doc = store.get_document_by_hash("h") assert doc is not None - assert doc["id"] == doc_id + assert doc["user_id"] == uid + +def test_add_chunks_are_retrievable_by_document(store: StoreProtocol) -> None: + """Chunks written by add_chunks come back via get_chunks_by_document, fields intact.""" + doc_id = uuid.uuid4() chunk_id = uuid.uuid4() - embedding = np.random.rand(settings.embedding_dimension).tolist() + store.add_document(id=doc_id, filename="d.md", file_hash="h") store.add_chunks( [ { "id": chunk_id, "document_id": doc_id, - "content": "This is a test chunk.", + "content": "hello", "metadata": {"page": 1}, - "embedding": embedding, + "embedding": [0.0] * settings.embedding_dimension, } ] ) - with store.get_connection() as conn: - cur = conn.execute("SELECT content, embedding FROM chunks WHERE id = %s", (chunk_id,)) - row = cur.fetchone() - assert row is not None - assert row["content"] == "This is a test chunk." - # Vector column round-trips: register_vector returns a numpy array. - assert row["embedding"].shape == (settings.embedding_dimension,) - assert np.allclose(row["embedding"], embedding) - + chunks = store.get_chunks_by_document(doc_id) -def test_add_document_is_idempotent(store: Store) -> None: - file_hash = "test-hash-" + str(uuid.uuid4()) - - first = store.add_document(id=uuid.uuid4(), filename="a.md", file_hash=file_hash) - # Same hash, different id + filename: the original row is kept, id is returned. - second = store.add_document(id=uuid.uuid4(), filename="b.md", file_hash=file_hash) - assert second == first - - doc = store.get_document_by_hash(file_hash) - assert doc is not None - assert doc["id"] == first - assert doc["filename"] == "b.md" + assert len(chunks) == 1 + assert chunks[0]["id"] == chunk_id + assert chunks[0]["content"] == "hello" + assert chunks[0]["metadata"] == {"page": 1} -def test_delete_document_cascades_chunks(store: Store) -> None: +def test_delete_document_cascades_chunks(store: StoreProtocol) -> None: + """Deleting a document removes it and cascades to its chunks.""" doc_id = uuid.uuid4() chunk_id = uuid.uuid4() - store.add_document(id=doc_id, filename="d.md", file_hash="test-hash-" + str(uuid.uuid4())) + store.add_document(id=doc_id, filename="d.md", file_hash="h") store.add_chunks( [ { @@ -74,13 +108,22 @@ def test_delete_document_cascades_chunks(store: Store) -> None: "document_id": doc_id, "content": "chunk", "metadata": {}, - "embedding": np.random.rand(settings.embedding_dimension).tolist(), + "embedding": [0.0] * settings.embedding_dimension, } ] ) + assert len(store.get_chunks_by_document(doc_id)) == 1 store.delete_document(doc_id) - with store.get_connection() as conn: - assert conn.execute("SELECT 1 FROM documents WHERE id = %s", (doc_id,)).fetchone() is None - assert conn.execute("SELECT 1 FROM chunks WHERE id = %s", (chunk_id,)).fetchone() is None + assert store.get_document_by_hash("h") is None + assert store.get_chunks_by_document(doc_id) == [] + + +def test_both_implementations_conform_to_protocol() -> None: + """Both Store and InMemoryStore satisfy StoreProtocol, structurally and at runtime.""" + # mypy --strict checks these assignments; the concrete Store constructs without connecting. + fake: StoreProtocol = InMemoryStore() + real: StoreProtocol = Store(database_url="postgresql://placeholder") + assert isinstance(fake, StoreProtocol) + assert isinstance(real, StoreProtocol) diff --git a/pyproject.toml b/pyproject.toml index c5a0b61..ec27294 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,4 +58,5 @@ ignore_missing_imports = true testpaths = ["packages"] markers = [ "integration: tests that require a live Postgres+pgvector instance (docker compose up -d)", + "unit: fast tests with no external services (auto-applied to every non-integration test)", ] From a325e8f1ea8b9f785e3c6ce29188a76d1a01664e Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Sat, 4 Jul 2026 18:47:35 -0400 Subject: [PATCH 14/19] #2 Enforced linting rule for docstring presence and made respective changes. --- docs/conf.py | 2 +- packages/rag_core/src/rag_core/config.py | 4 +++- packages/rag_core/src/rag_core/store/__init__.py | 2 ++ packages/rag_core/src/rag_core/store/client.py | 3 +++ packages/rag_core/src/rag_core/store/memory.py | 8 +++++++- packages/rag_core/tests/test_smoke.py | 1 + pyproject.toml | 5 ++++- 7 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index de1c0ea..63d1d74 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ """Sphinx configuration for the Study Assistant documentation site. Builds one static site from the all-Markdown docs/ tree (via myst-parser) plus an autodoc API -reference for rag_core (see ADR 0005). Build with: +reference for rag_core. Build with: uv run sphinx-build -b html docs site -W """ diff --git a/packages/rag_core/src/rag_core/config.py b/packages/rag_core/src/rag_core/config.py index d640e3c..3aab87c 100644 --- a/packages/rag_core/src/rag_core/config.py +++ b/packages/rag_core/src/rag_core/config.py @@ -1,3 +1,5 @@ +"""Application configuration for rag_core, loaded from the environment / ``.env``.""" + from uuid import UUID from pydantic_settings import BaseSettings, SettingsConfigDict @@ -17,7 +19,7 @@ class Settings(BaseSettings): # Vector dimensions (pinned to bge-m3) embedding_dimension: int = 1024 - # Default user for single-user local scope (ADR 0003) + # Default user for single-user local scope default_user_id: UUID = UUID("00000000-0000-0000-0000-000000000000") diff --git a/packages/rag_core/src/rag_core/store/__init__.py b/packages/rag_core/src/rag_core/store/__init__.py index 83ed2b2..225220c 100644 --- a/packages/rag_core/src/rag_core/store/__init__.py +++ b/packages/rag_core/src/rag_core/store/__init__.py @@ -1,3 +1,5 @@ +"""Store package: the ``StoreProtocol`` seam plus its DB-backed and in-memory implementations.""" + from rag_core.store.client import Store from rag_core.store.memory import InMemoryStore from rag_core.store.protocol import StoreProtocol diff --git a/packages/rag_core/src/rag_core/store/client.py b/packages/rag_core/src/rag_core/store/client.py index 14ca0c0..b8d594c 100644 --- a/packages/rag_core/src/rag_core/store/client.py +++ b/packages/rag_core/src/rag_core/store/client.py @@ -1,3 +1,5 @@ +"""The concrete pgvector-backed ``Store`` — the DB implementation of ``StoreProtocol``.""" + from typing import Any from uuid import UUID @@ -14,6 +16,7 @@ class Store: """Store client for pgvector-backed RAG storage.""" def __init__(self, database_url: str | None = None) -> None: + """Bind to ``database_url`` (or ``settings.database_url``); does not open a connection.""" url = database_url or settings.database_url if url is None: raise RuntimeError( diff --git a/packages/rag_core/src/rag_core/store/memory.py b/packages/rag_core/src/rag_core/store/memory.py index b6a541f..3eb5eb1 100644 --- a/packages/rag_core/src/rag_core/store/memory.py +++ b/packages/rag_core/src/rag_core/store/memory.py @@ -16,7 +16,7 @@ class InMemoryStore: """Dict-backed fake conforming to ``StoreProtocol``.""" def __init__(self) -> None: - # Keyed by document / chunk id. ``document`` rows mirror Store's SELECT shape. + """Start empty, keyed by document / chunk id (document rows mirror Store's SELECT shape).""" self.documents: dict[UUID, dict[str, Any]] = {} self.chunks: dict[UUID, dict[str, Any]] = {} @@ -30,6 +30,7 @@ def add_document( file_hash: str, user_id: UUID | None = None, ) -> UUID: + """Insert document metadata idempotently by file_hash, returning the stored id.""" uid = user_id or settings.default_user_id # Idempotent by file_hash: keep the original row/id, refresh the filename. for doc in self.documents.values(): @@ -46,11 +47,13 @@ def add_document( return id def update_document(self, id: UUID, filename: str) -> None: + """Update the stored filename; a no-op if the document is unknown.""" doc = self.documents.get(id) if doc is not None: doc["filename"] = filename def delete_document(self, id: UUID) -> None: + """Delete a document and cascade-drop its chunks.""" self.documents.pop(id, None) # Cascade: drop chunks belonging to the document. self.chunks = { @@ -58,6 +61,7 @@ def delete_document(self, id: UUID) -> None: } def get_document_by_hash(self, file_hash: str) -> dict[str, Any] | None: + """Return a copy of the document with this file_hash, or None if absent.""" for doc in self.documents.values(): if doc["file_hash"] == file_hash: return dict(doc) @@ -68,6 +72,7 @@ def add_chunks( chunks: list[dict[str, Any]], user_id: UUID | None = None, ) -> None: + """Store each chunk by id: ``{id, document_id, content, metadata, embedding}``.""" uid = user_id or settings.default_user_id for chunk in chunks: self.chunks[chunk["id"]] = { @@ -80,4 +85,5 @@ def add_chunks( } def get_chunks_by_document(self, document_id: UUID) -> list[dict[str, Any]]: + """Return copies of every chunk belonging to ``document_id``, or ``[]`` if none.""" return [dict(c) for c in self.chunks.values() if c["document_id"] == document_id] diff --git a/packages/rag_core/tests/test_smoke.py b/packages/rag_core/tests/test_smoke.py index df3f066..6e2d9a4 100644 --- a/packages/rag_core/tests/test_smoke.py +++ b/packages/rag_core/tests/test_smoke.py @@ -8,4 +8,5 @@ def test_package_imports() -> None: + """rag_core imports and exposes its version string.""" assert rag_core.__version__ == "0.0.0" diff --git a/pyproject.toml b/pyproject.toml index ec27294..408bc15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,10 @@ target-version = "py312" src = ["packages/rag_core/src"] [tool.ruff.lint] -select = ["E", "F", "I", "UP", "B"] +# D10x = docstring-presence rules (pydocstyle): every module, class, function, and __init__ must be +# documented, repo-wide. The stylistic D2xx/D4xx rules are deliberately left out so existing +# only presence of docstrings is enforced and not style. +select = ["E", "F", "I", "UP", "B", "D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107"] [tool.mypy] python_version = "3.12" From cf1bdd21381833b7576511ea7adec27928164bbb Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Sat, 4 Jul 2026 18:48:22 -0400 Subject: [PATCH 15/19] #2 Documentation updates in accordance with progress made for Phase 1. --- AGENTS.md | 45 +++++++++++++++++++++++-------------- CHEATSHEET.md | 2 +- README.md | 5 +++-- docs/reference/index.md | 16 +++++++++++++ packages/rag_core/AGENTS.md | 24 +++++++++++++++----- 5 files changed, 67 insertions(+), 25 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a543647..105f01a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ Anthropic API. - Fetch the current issue being worked on from the GitHub repo, read its full content. - Also read the full content of the current issue's sub issues (if any). -## Governance model +## Governance Canonical context lives in `AGENTS.md` files (tool-neutral). Each `CLAUDE.md` is a one-line `@AGENTS.md` import stub so Claude Code's directory-walk loading picks up the same content. Edit @@ -26,7 +26,25 @@ Canonical context lives in `AGENTS.md` files (tool-neutral). Each `CLAUDE.md` is surface; agents read the nearest file in the tree, so the closest one wins. This file is the root entry point — keep package-specific detail in the package's own `AGENTS.md`. -## Dev environment tips +## Development + +- Strict TDD, in three passes (binding, not a preference). For any new feature or issue: pass one + writes tests only — fully describing the expected behavior — with no interface or implementation + code, run and expect red. Pass two defines the interface(s) the code will satisfy — the `Protocol` + types and signatures, per the interface-first architecture + ([ADR 0006](docs/decisions/0006-interface-first-architecture.md)) — with no implementation logic. + Pass three adds the implementation behind those interfaces until the tests pass (green). This + applies to both unit and integration tests, per the `@pytest.mark.integration` marker in + `pyproject.toml`. +- As much as possible, write parametrized tests against the protocol. Use the `rag_core` module as + an example. +- Docstrings are mandatory on every module, class, and function repo-wide — including fixtures and + pytest hooks in `conftest.py`. This is machine-enforced: ruff's pydocstyle presence rules + (`D100`–`D107`) run in the same `ruff check` the CI gate uses, so a missing docstring fails the + build. A test's docstring states what behavior it pins; a fixture's states what it provides. Keep + them to one line unless the why is non-obvious. + +## Environment - Python 3.12 on a `uv` workspace. Run `uv sync` once to resolve and install every workspace member (root + `packages/*`) into a single `.venv`. @@ -43,24 +61,17 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. - Install hooks once with `pre-commit install`; `pre-commit run --all-files` runs lint + format + typecheck across the tree. -## Testing instructions +## Testing -- Strict TDD, in three passes (binding, not a preference). For any new feature or issue: pass one - writes tests only — fully describing the expected behavior — with no interface or implementation - code, run and expect red. Pass two defines the interface(s) the code will satisfy — the `Protocol` - types and signatures, per the interface-first architecture - ([ADR 0006](docs/decisions/0006-interface-first-architecture.md)) — with no implementation logic. - Pass three adds the implementation behind those interfaces until the tests pass (green). This - applies to both unit and integration tests, per the `@pytest.mark.integration` marker in - `pyproject.toml`. - The CI plan is in `.github/workflows/ci.yaml`: a `checks` job (ruff lint, ruff format check, mypy, pytest against a pgvector service) and a `docs` job that builds the Sphinx site (`uv run sphinx-build -b html docs site -W`, build-only — deployment lives in `docs-deploy.yaml`). - Run the whole suite with `uv run pytest`. Integration tests need the database — run `docker compose up -d` first or they fail to connect. -- Integration tests are marked `@pytest.mark.integration`. Scope a run with - `uv run pytest -m "not integration"` (fast, no DB) or `-m integration` (DB-backed); target one - test with `uv run pytest -k ""`. +- Integration tests are marked `@pytest.mark.integration`. Scope a run with `uv run pytest -m unit` + (fast, no DB) or `-m integration` (DB-backed); target one test with `uv run pytest -k ""`. + The `unit` marker is auto-applied to every non-integration test by the root `conftest.py`, so + `-m unit` always selects the full fast suite — mark only integration tests explicitly. - Integration tests run against a separate database (`TEST_DATABASE_URL`, e.g. `study_assistant_test`), never the app's `DATABASE_URL`, and truncate their tables between tests. `docker compose up -d` remains the only setup step — the test database is created automatically by @@ -77,20 +88,20 @@ entry point — keep package-specific detail in the package's own `AGENTS.md`. retrofit constraints, so run `docker compose down -v && docker compose up -d` after editing `store/schema.py`. -## Code style +## Style - `ruff` formats and lints (line length 100, rule set `E,F,I,UP,B`); `mypy` runs in `strict` mode. Both must pass — do not silence them without cause. - Type every function signature; `mypy --strict` rejects untyped defs. Prefer explicit, narrow types over `Any`. -## Binding decisions (ADRs) +## Decisions (ADRs) The locked architectural decisions are recorded as ADRs, indexed in [`docs/decisions/`](docs/decisions/). They are binding and the source of truth — read the relevant ADR before changing what it governs, and do not restate its rules here. -## PR instructions +## PRs - Branch from `dev`, naming the branch with its GitHub issue number first (e.g. `4-embeddings` for issue #4) so the branch links back to its issue. diff --git a/CHEATSHEET.md b/CHEATSHEET.md index 2aede90..e03722a 100644 --- a/CHEATSHEET.md +++ b/CHEATSHEET.md @@ -22,7 +22,7 @@ uv sync ```bash uv run pytest # everything -uv run pytest -m "not integration" # fast, no DB +uv run pytest -m unit # fast, no DB uv run pytest -m integration # DB-backed uv run pytest -k "" # one test ``` diff --git a/README.md b/README.md index c7b0bfe..7f746b3 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ free; only answer generation calls the Anthropic API. ## Status Early scaffolding (Phase 1 — Storage + Schema). The repo contains the `uv` workspace root, the -shared `packages/rag_core/` library with the pgvector storage layer, and the Postgres + pgvector dev -stack. Later phases add ingestion, retrieval logic, the CLI, MCP server, FastAPI backend, and React +shared `packages/rag_core/` library with the pgvector storage layer behind a `StoreProtocol` +interface (plus an in-memory fake for DB-free unit tests), and the Postgres + pgvector dev stack. +Later phases add ingestion, retrieval logic, the CLI, MCP server, FastAPI backend, and React frontend. ## Prerequisites diff --git a/docs/reference/index.md b/docs/reference/index.md index 7b81db3..78f34c7 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -17,6 +17,14 @@ :show-inheritance: ``` +## rag_core.store.protocol + +```{eval-rst} +.. automodule:: rag_core.store.protocol + :members: + :show-inheritance: +``` + ## rag_core.store.client ```{eval-rst} @@ -25,6 +33,14 @@ :show-inheritance: ``` +## rag_core.store.memory + +```{eval-rst} +.. automodule:: rag_core.store.memory + :members: + :show-inheritance: +``` + ## rag_core.store.schema ```{eval-rst} diff --git a/packages/rag_core/AGENTS.md b/packages/rag_core/AGENTS.md index f6c5b4c..5766cf9 100644 --- a/packages/rag_core/AGENTS.md +++ b/packages/rag_core/AGENTS.md @@ -12,12 +12,9 @@ per-model embedding columns (0008). ## Known drift from ADRs -Two accepted ADRs are ahead of the code — don't assume the shape below already matches the ADR just -because it's "Accepted": +One accepted ADR is still ahead of the code — don't assume the shape below already matches the ADR +just because it's "Accepted": -- **ADR 0006 (interface-first):** `store/__init__.py` exports the concrete `Store` class directly; - there is no `Protocol` in front of it yet. Store access is still concrete everywhere it's called. - Introducing the `Protocol` (plus a fake for unit tests) is open work, not done. - **ADR 0008 (per-model embedding columns):** `config.py` still reads a single global `settings.embedding_dimension`, and `schema.py` still builds one shared `embedding vector(N)` column — the pre-0008 shape ADR 0008 exists to replace. Explicit per-call dimensions and @@ -28,6 +25,23 @@ because it's "Accepted": Not ADR-level, but load-bearing for anyone adding to `store/`: +- Callers depend on `StoreProtocol` (ADR 0006), not the concrete `Store` — type new consumers + against the `Protocol` and inject the implementation. `InMemoryStore` is the fake for DB-free unit + tests; the DB-backed `Store` is exercised by the `@pytest.mark.integration` tests. Keep both + implementations and the `Protocol` in lockstep when you change a method signature — add a new + method in three passes (tests, then the `Protocol` signature + `NotImplementedError` stubs on both + implementations, then real logic), per the three-pass TDD rule in the root `AGENTS.md`. +- `get_connection()` is deliberately not on `StoreProtocol` — it returns a raw `psycopg` connection + (the driver dependency ADR 0006 keeps out of retrieval logic). Only the concrete `Store` and the + integration tests use it. `get_chunks_by_document` is the one read path chunks do have on the + Protocol; if a new read need comes up, prefer extending the Protocol over reaching for + `get_connection()` from outside `Store`. +- Test layout mirrors this split: `test_store.py` holds every behavior `StoreProtocol` + guarantees, parametrized over the `store` fixture (`InMemoryStore` and `Store`, labeled + `[memory]`/`[db]`) so the two implementations can't silently drift apart. `test_db_store.py` holds + only what's genuinely Store-only and unexpressible through the Protocol (raw vector round-tripping, + schema DDL idempotency), via the DB-only `db_store` fixture. A new `StoreProtocol` method's tests + belong in `test_store.py` against `store`, not duplicated per-implementation. - Every write method takes an optional `user_id: UUID | None`, defaulting to `settings.default_user_id` (the ADR 0003 seam). New methods should follow the same signature rather than hardcoding the default user or dropping the parameter. From a626708f84de46a0e35b5c71fb1f8a8840af00d7 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Tue, 7 Jul 2026 12:07:34 -0400 Subject: [PATCH 16/19] Fix pre commit to run uv workspace so libraries resolve. --- .pre-commit-config.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6fc33d..860786e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,10 +20,14 @@ repos: args: [--fix] - id: ruff-format - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.1 + - repo: local hooks: - id: mypy - # Match the workspace config; no third-party deps to install yet. - args: [--config-file=pyproject.toml] + name: mypy + # Run the workspace mypy from the uv venv (same as CI's `uv run mypy`) so it resolves + # psycopg / numpy / pytest / pydantic-settings. An isolated mirrors-mypy env can't see those + # third-party deps and fails on import-not-found even when the code typechecks clean. + entry: uv run mypy + language: system + types: [python] pass_filenames: false From c1d6ba8af021ab05ed1a3d2d7a0ab82527e9da56 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Tue, 7 Jul 2026 13:42:56 -0400 Subject: [PATCH 17/19] Updated PR template to add instructions as comments for agents drafting PRs. --- .github/PULL_REQUEST_TEMPLATE.md | 39 ++++++++++++++++++- .../rag_core/src/rag_core/store/schema.py | 2 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1ea5de4..bea26f0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,17 +1,52 @@ -# [PR Title] - ## Summary + + ## Motivation + + ## Type + + - [ ] Bug - [ ] Feature - [ ] Improvement ## Related issues + + ## Verification + + ## Decision records / ADRs + + diff --git a/packages/rag_core/src/rag_core/store/schema.py b/packages/rag_core/src/rag_core/store/schema.py index 979ef6c..7849599 100644 --- a/packages/rag_core/src/rag_core/store/schema.py +++ b/packages/rag_core/src/rag_core/store/schema.py @@ -38,7 +38,7 @@ ); -- HNSW index for vector similarity search (cosine distance) -CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON chunks +CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON chunks USING hnsw (embedding vector_cosine_ops); -- GIN index for full-text search (BM25) From a1410ac7fffb832123bbf35f531aaa5d4bff3495 Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Tue, 7 Jul 2026 16:48:34 -0400 Subject: [PATCH 18/19] Update CodeRabbit configuration and add PR Agent workflow for automated reviews. --- .coderabbit.yaml | 15 ++------------- .github/workflows/pr_agent.yaml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/pr_agent.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml index e65ce24..a54f132 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -5,20 +5,9 @@ language: en-US reviews: profile: chill # fewer, higher-signal comments (the alternative is "assertive") - high_level_summary: true + high_level_summary: false poem: false auto_review: enabled: true - drafts: false + drafts: true base_branches: [".*"] - path_instructions: - - path: "**/*.py" - instructions: >- - Python 3.12, typed under `mypy --strict` and linted by ruff (rules E,F,I,UP,B, - line length 100). Enforce the project conventions in AGENTS.md and docs/decisions: - retrieval logic lives only in `rag_core`; no top-level `torch` import outside - `embed/` and `rerank/`; relevant tables carry the defaulted `user_id` column. - -tools: - ruff: - enabled: true diff --git a/.github/workflows/pr_agent.yaml b/.github/workflows/pr_agent.yaml new file mode 100644 index 0000000..682539b --- /dev/null +++ b/.github/workflows/pr_agent.yaml @@ -0,0 +1,30 @@ +name: PR Agent (Gemini) + +on: + pull_request: + types: [opened, reopened, ready_for_review] + issue_comment: + +jobs: + pr_agent_job: + if: ${{ github.event.sender.type != 'Bot' }} + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + contents: write + checks: write + name: Run PR Agent (Gemini free tier) on pull requests and comments + steps: + - name: PR Agent action step + uses: the-pr-agent/pr-agent@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Free-tier Gemini models via Google AI Studio — no OpenAI cost. + config.model: "gemini/gemini-2.5-flash" + config.fallback_models: '["gemini/gemini-2.5-flash-lite"]' + GOOGLE_AI_STUDIO.GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # Auto-run on new/updated PRs. + github_action_config.auto_review: "true" + github_action_config.auto_describe: "true" + github_action_config.auto_improve: "true" From f6ebe965c42388aa1fda1a74fc6176138ac72d5d Mon Sep 17 00:00:00 2001 From: Sidharth Anandkumar Date: Tue, 7 Jul 2026 17:09:27 -0400 Subject: [PATCH 19/19] Update CodeRabbit configuration to skip draft reviews and prevent re-review on new commits --- .coderabbit.yaml | 3 ++- .github/workflows/pr_agent.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index a54f132..15521b9 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -9,5 +9,6 @@ reviews: poem: false auto_review: enabled: true - drafts: true + drafts: false # skip drafts; only review once the PR is published/marked ready + auto_incremental_review: false # don't re-review on every new commit base_branches: [".*"] diff --git a/.github/workflows/pr_agent.yaml b/.github/workflows/pr_agent.yaml index 682539b..8e8f29d 100644 --- a/.github/workflows/pr_agent.yaml +++ b/.github/workflows/pr_agent.yaml @@ -7,7 +7,7 @@ on: jobs: pr_agent_job: - if: ${{ github.event.sender.type != 'Bot' }} + if: ${{ github.event.sender.type != 'Bot' && !github.event.pull_request.draft }} runs-on: ubuntu-latest permissions: issues: write