diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5983d9b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.git +.github +.pytest_cache +.venv +.coverage* +**/__pycache__ +**/*.pyc +dist +htmlcov +plan +src/tests +website +_prompts +.loopy_loop +.eval-banana diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..70f7d954 --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +# Local-only quickstart credentials. Replace every secret before exposing the stack. +UGM_POSTGRES_USER=ugm +UGM_POSTGRES_PASSWORD=ugm-local-only +UGM_POSTGRES_DB=ugm +UGM_MINIO_ACCESS_KEY=remember-local +UGM_MINIO_SECRET_KEY=remember-local-only + +# One stable deployment identity; changing it creates a different trust domain. +UGM_SELFHOST_DEPLOYMENT_ID=11111111-1111-4111-8111-111111111111 +UGM_SELFHOST_DEPLOYMENT_SLUG=local +UGM_SELFHOST_DEPLOYMENT_NAME=Local memory +UGM_SELFHOST_API_PORT=8000 + +# Required to start the provider adapter. The short Markdown smoke path makes no +# provider request; replace this value before processing a real corpus. +UGM_OPENROUTER_API_KEY=replace-before-real-use diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 696d7bf4..955a144d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,66 @@ concurrency: cancel-in-progress: true jobs: + compose-quickstart: + name: Compose quickstart + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Validate Compose model + run: docker compose --env-file .env.example config --quiet + + - name: Start fresh self-host deployment + run: docker compose --env-file .env.example up --build --detach --wait + + - name: Prove MinIO conditional create + run: | + docker compose --env-file .env.example exec -T api python - <<'PY' + from ultimate_memory.adapters.selfhost import MinIOObjectStore, MinIOSettings + from ultimate_memory.model import ObjectAlreadyExistsError, ObjectKey + + store = MinIOObjectStore( + bucket="remember-raw", settings=MinIOSettings.model_validate({}) + ) + key = ObjectKey("ci/immutable-object") + store.write_bytes(key=key, content=b"first") + try: + store.write_bytes(key=key, content=b"replacement") + except ObjectAlreadyExistsError: + pass + else: + raise RuntimeError("MinIO replaced an immutable object key") + PY + + - name: Ingest and observe the E0 chain + run: | + curl --fail --data-binary $'# Hello\n\nCompose smoke.\n' \ + 'http://localhost:8000/ingest?filename=smoke.md&mime=text%2Fmarkdown' + observed='' + for attempt in {1..30}; do + observed="$(docker compose --env-file .env.example exec -T postgres \ + psql -U ugm -d ugm -Atc \ + "SELECT count(*) FILTER (WHERE stage='convert' AND status='succeeded'), count(*) FILTER (WHERE stage='structure' AND status='succeeded'), count(*) FILTER (WHERE stage='chunk' AND status='pending') FROM processing_state")" + if [ "$observed" = '1|1|1' ]; then + break + fi + sleep 1 + done + test "$observed" = '1|1|1' + test "$(docker compose --env-file .env.example exec -T postgres \ + psql -U ugm -d ugm -Atc 'SELECT count(*) FROM cost_ledger')" = '0' + + - name: Show service logs after failure + if: failure() + run: docker compose --env-file .env.example logs --no-color + + - name: Remove quickstart deployment + if: always() + run: docker compose --env-file .env.example down --volumes + checks: runs-on: ubuntu-latest permissions: diff --git a/.gitignore b/.gitignore index 16cf0083..e1f34c30 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .idea/ _additional_context/ _feature_planning/ +.env # python .venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..24e89b78 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM ghcr.io/astral-sh/uv:0.11.31@sha256:ecd4de2f060c64bea0ff8ecb182ddf46ba3fcccdc8a60cfdbaf20d1a047d7437 AS uv + +FROM python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 + +COPY --from=uv /uv /uvx /bin/ + +ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONUNBUFFERED=1 \ + UV_LINK_MODE=copy + +WORKDIR /app + +COPY pyproject.toml uv.lock README.md LICENSE alembic.ini ./ + +RUN addgroup --system app \ + && adduser --system --ingroup app app \ + && mkdir -p /var/lib/ultimate-memory/forget-manifests \ + && chown -R app:app /var/lib/ultimate-memory \ + && uv sync --locked --no-dev --extra server --no-install-project + +COPY src ./src + +RUN uv sync --locked --no-dev --extra server + +USER app + +ENTRYPOINT ["python", "-m", "ultimate_memory.profiles.selfhost"] +CMD ["api"] diff --git a/README.md b/README.md index 883a78cc..e5b7bbbd 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ distill them into progressively more abstract, navigable knowledge — while kee auditable by humans. Scale is a requirement, not an aspiration: it is meant to still be useful at a million documents. -> **⚠️ Implementation has just begun; there is nothing usable yet.** The design came first and -> lives here in full — requirements, architecture, research, decisions, and the open questions. -> The build follows [plan/plans/roadmap.md](plan/plans/roadmap.md) (phase 0 is under way). If -> you're looking for a working library, it isn't here yet. +> **Pre-release software.** Phases 0–6 and most of Phase 7 are implemented and tested, but no +> public package or container release exists yet. The fresh-deployment Docker Compose skeleton +> is documented under [Self-host deployment](website/src/app/docs/deployment/page.mdx); it proves +> PostgreSQL, MinIO, API ingestion, and the first two E0 worker stages, not a production rollout. +> The build follows [plan/plans/roadmap.md](plan/plans/roadmap.md). ## TL;DR diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..8081c563 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,98 @@ +name: ultimate-memory + +x-app: &app + build: + context: . + dockerfile: Dockerfile + image: ultimate-memory:local + environment: + UGM_DATABASE_URL: postgresql+psycopg://${UGM_POSTGRES_USER}:${UGM_POSTGRES_PASSWORD}@postgres:5432/${UGM_POSTGRES_DB} + UGM_MINIO_ENDPOINT_URL: http://minio:9000 + UGM_MINIO_ACCESS_KEY: ${UGM_MINIO_ACCESS_KEY} + UGM_MINIO_SECRET_KEY: ${UGM_MINIO_SECRET_KEY} + UGM_OPENROUTER_API_KEY: ${UGM_OPENROUTER_API_KEY} + UGM_SELFHOST_DEPLOYMENT_ID: ${UGM_SELFHOST_DEPLOYMENT_ID} + UGM_SELFHOST_DEPLOYMENT_SLUG: ${UGM_SELFHOST_DEPLOYMENT_SLUG} + UGM_SELFHOST_DEPLOYMENT_NAME: ${UGM_SELFHOST_DEPLOYMENT_NAME} + UGM_SELFHOST_API_PORT: "8000" + volumes: + - app-state:/var/lib/ultimate-memory + - forget-manifests:/var/lib/ultimate-memory/forget-manifests + +services: + postgres: + image: ghcr.io/dbsystel/postgresql-partman@sha256:bf9d2331aaeb3e33a4e6e73b4c024ca618d4d1ea050bbab978125ff63b83385a + environment: + POSTGRES_USER: ${UGM_POSTGRES_USER} + POSTGRES_PASSWORD: ${UGM_POSTGRES_PASSWORD} + POSTGRES_DB: ${UGM_POSTGRES_DB} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${UGM_POSTGRES_USER} -d ${UGM_POSTGRES_DB}"] + interval: 2s + timeout: 5s + retries: 30 + volumes: + - postgres-data:/var/lib/postgresql/data + + minio: + image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: server /data --console-address :9001 + environment: + MINIO_ROOT_USER: ${UGM_MINIO_ACCESS_KEY} + MINIO_ROOT_PASSWORD: ${UGM_MINIO_SECRET_KEY} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 2s + timeout: 5s + retries: 30 + ports: + - "9001:9001" + volumes: + - minio-data:/data + + setup: + <<: *app + command: ["setup"] + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + + api: + <<: *app + command: ["api"] + depends_on: + setup: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz', timeout=2).read()" + interval: 10s + timeout: 5s + retries: 30 + ports: + - "${UGM_SELFHOST_API_PORT}:8000" + + worker-convert: + <<: *app + command: ["worker", "--stage", "convert"] + depends_on: + setup: + condition: service_completed_successfully + + worker-structure: + <<: *app + command: ["worker", "--stage", "structure"] + depends_on: + setup: + condition: service_completed_successfully + +volumes: + app-state: + forget-manifests: + minio-data: + postgres-data: diff --git a/plan/plans/phase-0-foundations.md b/plan/plans/phase-0-foundations.md index f381e5b9..b3be71d7 100644 --- a/plan/plans/phase-0-foundations.md +++ b/plan/plans/phase-0-foundations.md @@ -24,7 +24,7 @@ scaffold exists with ≥1 seeded doc. | WP-0.4 | **The D61 port interfaces** (`ports/` Protocols: object store, task queue, mounts, git remote, model provider, telemetry, auth) + import-linter contracts in CI | packaging §3–4; D61, D62 | WP-0.1 | `ports/` + CI architecture checks | illegal import fails CI (proven by a deliberate violation) | done | | WP-0.4a | **Self-host adapters**: pg-queue delivery shell (`LISTEN/NOTIFY` + `SKIP LOCKED`, transactional enqueue, token-bucket rate limits), local-FS object store, local mount publisher, `adapters/testing` tier (MinIO wiring lands with WP-0.4c) | packaging §3, §5; D62 | WP-0.4, WP-0.3 | `adapters/selfhost` + `adapters/testing` | demo chain runs against real Postgres with zero GCP deps; transactional-enqueue crash test | done | | WP-0.4b | **Reference adapters**: Cloud Tasks push shell + dispatch server, GCS store, gcsfuse publisher; **the janitor sweep** (shared, port-agnostic) | packaging §3; orchestration §2–3; D61 | WP-0.4 | `adapters/gcp` + janitor job | same demo chain on the GCP profile; janitor re-announces a killed delivery on BOTH profiles | planned | -| WP-0.4c | **Compose self-host profile** (postgres + minio + api + worker; `profiles/selfhost`) — the quickstart skeleton | packaging §5 | WP-0.4a | docker-compose + profile module | `docker compose up` → demo ingest → state rows; CI-run | planned | +| WP-0.4c | **Compose self-host profile** (postgres + minio + api + worker; `profiles/selfhost`) — the quickstart skeleton | packaging §5 | WP-0.4a | docker-compose + profile module | `docker compose up` → demo ingest → state rows; CI-run | done | | WP-0.5 | **Eval harness skeleton** (questions #14 — this WP owns it): golden-set storage (`golden_pairs`, `golden_claim_labels`, `canary_cases`, `eval_runs`), suite runner, CI wiring | schema §5; D22; registries §10 | WP-0.2 | harness package + `eval` CI job | empty suites run; a seeded canary fails deliberately and blocks CI | done | | WP-0.6 | Golden-set labeling tooling (LLM-propose / human-adjudicate loop, circularity guard) | D22; registries §11.1 | WP-0.5 | labeling CLI | 20 seed pairs labeled end-to-end | planned | | WP-0.7 | Blockizer golden corpus scaffold (expected block-hash regression per `blockizer_version`) | e1 §2 (D57) | WP-0.5 | corpus + CI check | seeded doc's hashes locked; a deliberate parser change trips CI | done | @@ -99,6 +99,17 @@ enqueue delivers exactly its row's wake — the schema trigger's by-construction announce re-delivery, and the demo chain draining through the loop with zero GCP dependencies. MinIO wiring lands with WP-0.4c per the sequencing amendment. +**WP-0.4c complete (2026-07-22):** the fresh-deployment Compose skeleton wires the +real `profiles/selfhost` composition to pinned PostgreSQL 16 + pg_partman and MinIO images, +one migration/bootstrap service, the HTTP API, and separate `convert`/`structure` workers. +The MinIO adapter preserves write-once object semantics, path boundaries, routing metadata, +and hard-forget purge verification. A real Compose acceptance started from empty named volumes, +served health + recipes, ingested Markdown through MinIO, and observed `convert=succeeded`, +`structure=succeeded`, and `chunk=pending` in the authoritative work ledger with zero cost rows +(no provider call). CI repeats that proof in a job parallel to the Python matrix. This remains a +fresh, pre-release infrastructure skeleton: later-stage workers, authentication, restore +recovery, published GHCR images, and the public rename stay in their owning release work. + **WP-0.5 + WP-0.7 complete (2026-07-18):** the eval-harness skeleton lands in `eval/harness.py` (suites over `canary_cases`, runs recorded in `eval_runs`, per-suite evaluator registration; @@ -116,7 +127,7 @@ the edited block, offsets slice document.md exactly. cleanly (WP-0.2); the no-op worker runs end-to-end through the task-queue delivery port with `processing_state` + `cost_ledger` rows (WP-0.3 + WP-0.4a, per the sequencing amendment); the eval harness runs empty suites in CI (WP-0.5); the blockizer golden corpus exists with a seeded -doc (WP-0.7). Carried forward, not exit-blocking: WP-0.4b/0.4c (GCP parity + compose, +doc (WP-0.7). Carried forward, not exit-blocking: WP-0.4b (GCP parity, Phase-1-parallel) and WP-0.6 (golden-set labeling tooling, needed by Phase 2's measured thresholds). diff --git a/plan/plans/phase-5-retrieval-complete.md b/plan/plans/phase-5-retrieval-complete.md index 001f071a..c3160cc7 100644 --- a/plan/plans/phase-5-retrieval-complete.md +++ b/plan/plans/phase-5-retrieval-complete.md @@ -16,4 +16,4 @@ skill; latency budget measured against the §10 envelope. | WP-5.4 | Surfaces: MCP (rendered from registry), CLI parity, API auth perimeter hooks | retrieval §7; D50–D51 | WP-5.2 | MCP server + CLI | tool list = registry; parity test | done (PR #108; `RecipeSurface`/`RecipeMcpServer` + API `/recipes`+`/recipe/{name}`+auth + `ugm query` CLI; parity test + reference docs) | | WP-5.5 | **Consumption skill v1** (per-deployment rendered) + the S58 protocol as a repeatable eval | retrieval §8; D51 | WP-5.3 | skill + S58 harness | S58 green with a cold harness | done (PR #109; deployment/scopes/K/mount/recipe-rendered `SKILL.md`; provider-backed S58 retrieval canary) | | WP-5.6 | Retrieval spikes: Lance scale, hub pagination, rerank weights, envelope overhead, hydration batching, resolve context | retrieval §13 | WP-5.3 | spike reports + tuned constants | recorded in eval_runs | done (PR #110; six-spike battery recorded at 10M Lance rows and a 100k-edge hub; measured retrieval defaults and regressions implemented) | -| WP-5.7 | **PyPI packaging of the client surface**: base install = SDK + CLI + MCP; extras `[server]`/`[connectors-*]`/`[k]`; **lineage-aware ingest** in SDK/CLI; connector-management commands | packaging §1–2; D62 | WP-5.4 | the pip package (dist `remember-dev` — decided, questions.md §11a) | fresh-venv install → query + ingest against a compose deployment; push-feeder lineage test (stable source_ref → versions) | done (PR #111; client-first wheel + SDK/CLI/remote MCP, lineage-aware E0 ingest, and connector-management contracts; exact compose replay remains coupled to carried WP-0.4c) | +| WP-5.7 | **PyPI packaging of the client surface**: base install = SDK + CLI + MCP; extras `[server]`/`[connectors-*]`/`[k]`; **lineage-aware ingest** in SDK/CLI; connector-management commands | packaging §1–2; D62 | WP-5.4 | the pip package (dist `remember-dev` — decided, questions.md §11a) | fresh-venv install → query + ingest against a compose deployment; push-feeder lineage test (stable source_ref → versions) | done (PR #111; client-first wheel + SDK/CLI/remote MCP, lineage-aware E0 ingest, and connector-management contracts; the Compose deployment proof is now supplied by completed WP-0.4c) | diff --git a/plan/plans/roadmap.md b/plan/plans/roadmap.md index d3e4a260..5e526d71 100644 --- a/plan/plans/roadmap.md +++ b/plan/plans/roadmap.md @@ -95,7 +95,7 @@ needs to restate it: | Phase | Name | Builds | Keyed to designs | Status | |---|---|---|---|---| -| 0 | Foundations + harness | scaffolding, migrations, tenancy, `processing_state`/`cost_ledger`, queues, **eval harness + golden-set tooling**, blockizer golden corpus | schema; orchestration §1–2; D22 | done (exit criteria met 2026-07-18; WP-0.4b/0.4c + WP-0.6 carried per the phase file) | +| 0 | Foundations + harness | scaffolding, migrations, tenancy, `processing_state`/`cost_ledger`, queues, **eval harness + golden-set tooling**, blockizer golden corpus | schema; orchestration §1–2; D22 | done (exit criteria met 2026-07-18; WP-0.4c completed 2026-07-22; WP-0.4b + WP-0.6 remain carried per the phase file) | | 1 | Walking skeleton | one document end-to-end, everything minimal; 4 retrieval primitives + envelope core | e0, e1, e2_e3, observations, retrieval §2–3 | done (exit criteria met 2026-07-18; PRs #81-#87 — see the phase file) | | 2 | Truth machinery | full ER cascade + registries + review queue; supersession; observation adjudication; thresholds measured | registries, e2_e3 §5, observations | done (exit criteria met 2026-07-19; PRs #88-#94 — see the phase file) | | 3 | Evidence lifecycle | lineages/versions, Drive connector + sync cycles, currency + counting + reconciliation, chunk reuse, deletion grains, full PageIndex route | evidence_lifecycle, e1 §7, e0 | done (exit criteria met 2026-07-19; PRs #95-#99 — see the phase file) | diff --git a/pyproject.toml b/pyproject.toml index 81b49774..56659528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ [project.optional-dependencies] server = [ "alembic>=1.16", + "boto3>=1.42", "fastapi>=0.139.2", "ladybug>=0.18.2", "lancedb>=0.34.0", @@ -50,6 +51,7 @@ server = [ "onnxruntime>=1.24.1; python_version >= '3.14'", "psycopg[binary]>=3.2", "sqlalchemy>=2.0", + "uvicorn>=0.37", ] # The watched-directory connector uses only the standard library. Other # connector extras are added with their adapters, rather than speculatively. @@ -68,6 +70,7 @@ Issues = "https://github.com/writeitai/ultimate-memory/issues" [dependency-groups] dev = [ "alembic>=1.16", + "boto3>=1.42", "fastapi>=0.139.2", "import-linter>=2.13", "ladybug>=0.18.2", @@ -82,6 +85,7 @@ dev = [ "pytest-cov>=6.0", "ruff>=0.4", "sqlalchemy>=2.0", + "uvicorn>=0.37", ] [tool.hatch.build.targets.wheel] diff --git a/src/tests/adapters/test_minio_store.py b/src/tests/adapters/test_minio_store.py new file mode 100644 index 00000000..251f1a7c --- /dev/null +++ b/src/tests/adapters/test_minio_store.py @@ -0,0 +1,159 @@ +"""WP-0.4c contract tests for immutable MinIO object storage.""" + +from io import BytesIO + +from botocore.exceptions import ClientError +import pytest + +from ultimate_memory.adapters.selfhost import MinIOObjectStore +from ultimate_memory.adapters.selfhost.minio import _GetObjectOutput +from ultimate_memory.adapters.selfhost.minio import _HeadObjectOutput +from ultimate_memory.adapters.selfhost.minio import _ListObjectsOutput +from ultimate_memory.model import ObjectAlreadyExistsError +from ultimate_memory.model import ObjectKey +from ultimate_memory.model import ObjectKeyEscapesRootError + + +class _Body: + """A close-observable response body for the read contract.""" + + def __init__(self, *, content: bytes) -> None: + """Retain content and an initially open state.""" + self._stream = BytesIO(content) + self.closed = False + + def read(self) -> bytes: + """Read all remaining bytes.""" + return self._stream.read() + + def close(self) -> None: + """Record connection release.""" + self.closed = True + self._stream.close() + + +class _MemoryS3: + """The exact S3 client seam exercised by the adapter tests.""" + + def __init__(self) -> None: + """Start with no provisioned buckets or objects.""" + self.buckets: set[str] = set() + self.objects: dict[tuple[str, str], tuple[bytes, dict[str, str]]] = {} + self.last_body: _Body | None = None + + def head_bucket(self, *, Bucket: str) -> object: + """Raise the provider's ordinary absence code for a missing bucket.""" + if Bucket not in self.buckets: + raise _client_error(code="NoSuchBucket", operation="HeadBucket") + return {} + + def create_bucket(self, *, Bucket: str) -> object: + """Create a bucket once.""" + self.buckets.add(Bucket) + return {} + + def get_object(self, *, Bucket: str, Key: str) -> _GetObjectOutput: + """Return one streaming body.""" + body = _Body(content=self.objects[(Bucket, Key)][0]) + self.last_body = body + return {"Body": body} + + def put_object( + self, + *, + Bucket: str, + Key: str, + Body: bytes, + IfNoneMatch: str, + Metadata: dict[str, str], + ) -> object: + """Honor the S3 conditional-create header.""" + assert IfNoneMatch == "*" + identity = (Bucket, Key) + if identity in self.objects: + raise _client_error(code="PreconditionFailed", operation="PutObject") + self.objects[identity] = (Body, Metadata) + return {} + + def head_object(self, *, Bucket: str, Key: str) -> _HeadObjectOutput: + """Return metadata or the provider's ordinary absence code.""" + try: + _, metadata = self.objects[(Bucket, Key)] + except KeyError as error: + raise _client_error(code="NoSuchKey", operation="HeadObject") from error + return {"Metadata": metadata} + + def delete_object(self, *, Bucket: str, Key: str) -> object: + """Delete one object idempotently.""" + self.objects.pop((Bucket, Key), None) + return {} + + def list_objects_v2( + self, *, Bucket: str, Prefix: str, ContinuationToken: str = "" + ) -> _ListObjectsOutput: + """Return one deterministic page; the test corpus never paginates.""" + assert ContinuationToken == "" + return { + "Contents": [ + {"Key": key} + for bucket, key in sorted(self.objects) + if bucket == Bucket and key.startswith(Prefix) + ], + "IsTruncated": False, + } + + +def test_bucket_provision_and_immutable_round_trip() -> None: + """The adapter provisions once, records routing, and refuses replacement.""" + client = _MemoryS3() + store = MinIOObjectStore(bucket="raw", client=client) + + store.ensure_bucket() + store.ensure_bucket() + store.write_bytes( + key=ObjectKey("documents/a.md"), content=b"first", storage_class="cold" + ) + + assert client.buckets == {"raw"} + assert store.read_bytes(key=ObjectKey("documents/a.md")) == b"first" + assert client.last_body is not None and client.last_body.closed + assert store.storage_class_of(key=ObjectKey("documents/a.md")) == "cold" + with pytest.raises(ObjectAlreadyExistsError): + store.write_bytes(key=ObjectKey("documents/a.md"), content=b"replacement") + assert store.read_bytes(key=ObjectKey("documents/a.md")) == b"first" + + +def test_purge_respects_prefix_boundaries_and_verifies() -> None: + """A prefix purge removes descendants without touching sibling prefixes.""" + client = _MemoryS3() + store = MinIOObjectStore(bucket="artifacts", client=client) + store.ensure_bucket() + for name in ("doc/a", "doc/nested/b", "document/sibling", "exact"): + store.write_bytes(key=ObjectKey(name), content=name.encode()) + + store.purge_objects(keys=(ObjectKey("exact"),), prefixes=(ObjectKey("doc"),)) + store.verify_objects_purged( + keys=(ObjectKey("exact"),), prefixes=(ObjectKey("doc"),) + ) + + assert store.read_bytes(key=ObjectKey("document/sibling")) == b"document/sibling" + store.write_bytes(key=ObjectKey("doc/reappeared"), content=b"unsafe") + with pytest.raises(RuntimeError, match="doc/reappeared"): + store.verify_objects_purged(keys=(), prefixes=(ObjectKey("doc"),)) + + +@pytest.mark.parametrize("value", ("/absolute", "safe/../escape")) +def test_keys_cannot_escape_the_logical_store_root(value: str) -> None: + """MinIO applies the same traversal boundary as the local-FS adapter.""" + store = MinIOObjectStore(bucket="raw", client=_MemoryS3()) + + with pytest.raises(ObjectKeyEscapesRootError): + store.write_bytes(key=ObjectKey(value), content=b"no") + + +def _client_error(*, code: str, operation: str) -> ClientError: + """Construct a real botocore error so exception handling stays production-like.""" + return ClientError( + error_response={"Error": {"Code": code, "Message": code}}, + operation_name=operation, + ) diff --git a/src/ultimate_memory/adapters/__init__.py b/src/ultimate_memory/adapters/__init__.py index e6064d7b..422b250a 100644 --- a/src/ultimate_memory/adapters/__init__.py +++ b/src/ultimate_memory/adapters/__init__.py @@ -1,15 +1,21 @@ """Provider adapter package.""" +from typing import TYPE_CHECKING + from ultimate_memory.adapters.codex_writer import CodexAgentAdapterSettings from ultimate_memory.adapters.codex_writer import CodexCLIAgentAdapter from ultimate_memory.adapters.codex_writer import CodexCLIWriterAdapter from ultimate_memory.adapters.codex_writer import CodexWriterAdapterSettings -from ultimate_memory.adapters.markitdown_converter import MARKITDOWN_CONVERTER_VERSION -from ultimate_memory.adapters.markitdown_converter import MarkitdownConverter from ultimate_memory.adapters.openrouter import OpenRouterModelProvider from ultimate_memory.adapters.openrouter import OpenRouterProviderError from ultimate_memory.adapters.openrouter import OpenRouterSettings +if TYPE_CHECKING: + from ultimate_memory.adapters.markitdown_converter import ( + MARKITDOWN_CONVERTER_VERSION, + ) + from ultimate_memory.adapters.markitdown_converter import MarkitdownConverter + __all__ = ( "CodexCLIAgentAdapter", "CodexCLIWriterAdapter", @@ -21,3 +27,18 @@ "OpenRouterProviderError", "OpenRouterSettings", ) + + +def __getattr__(name: str) -> object: + """Load the media converter only for processes that actually compose it.""" + if name == "MARKITDOWN_CONVERTER_VERSION": + from ultimate_memory.adapters.markitdown_converter import ( + MARKITDOWN_CONVERTER_VERSION, + ) + + return MARKITDOWN_CONVERTER_VERSION + if name == "MarkitdownConverter": + from ultimate_memory.adapters.markitdown_converter import MarkitdownConverter + + return MarkitdownConverter + raise AttributeError(name) diff --git a/src/ultimate_memory/adapters/selfhost/__init__.py b/src/ultimate_memory/adapters/selfhost/__init__.py index 9b835954..59c9a935 100644 --- a/src/ultimate_memory/adapters/selfhost/__init__.py +++ b/src/ultimate_memory/adapters/selfhost/__init__.py @@ -1,8 +1,11 @@ """Self-host adapters: pg delivery shell, local-FS object store, local mounts (WP-0.4a).""" +from typing import TYPE_CHECKING + from ultimate_memory.adapters.selfhost.forget import LocalFSForgetManifestStore from ultimate_memory.adapters.selfhost.git import LocalGitRepository -from ultimate_memory.adapters.selfhost.lance import LanceChunkIndex +from ultimate_memory.adapters.selfhost.minio import MinIOObjectStore +from ultimate_memory.adapters.selfhost.minio import MinIOSettings from ultimate_memory.adapters.selfhost.mounts import AuditedRawReader from ultimate_memory.adapters.selfhost.mounts import LocalMountPublisher from ultimate_memory.adapters.selfhost.mounts import RawAccessDenied @@ -17,6 +20,9 @@ from ultimate_memory.adapters.selfhost.telemetry import JsonLineTelemetry from ultimate_memory.adapters.selfhost.watcher import LocalDirectoryWatcher +if TYPE_CHECKING: + from ultimate_memory.adapters.selfhost.lance import LanceChunkIndex + __all__ = ( "LanceChunkIndex", "LocalFSForgetManifestStore", @@ -26,6 +32,8 @@ "LocalFSObjectStore", "AuditedRawReader", "LocalMountPublisher", + "MinIOObjectStore", + "MinIOSettings", "RawAccessDenied", "storage_class_for", "ObjectAlreadyExistsError", @@ -35,3 +43,12 @@ "SelfHostWorkerLoop", "TokenBucket", ) + + +def __getattr__(name: str) -> object: + """Load the heavy LanceDB adapter only when a composition actually needs it.""" + if name == "LanceChunkIndex": + from ultimate_memory.adapters.selfhost.lance import LanceChunkIndex + + return LanceChunkIndex + raise AttributeError(name) diff --git a/src/ultimate_memory/adapters/selfhost/minio.py b/src/ultimate_memory/adapters/selfhost/minio.py new file mode 100644 index 00000000..36d4f6bf --- /dev/null +++ b/src/ultimate_memory/adapters/selfhost/minio.py @@ -0,0 +1,279 @@ +"""S3-compatible MinIO object storage for the self-host profile.""" + +from typing import cast +from typing import NotRequired +from typing import Protocol +from typing import TypedDict + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError +from pydantic import SecretStr +from pydantic_settings import BaseSettings +from pydantic_settings import SettingsConfigDict + +from ultimate_memory.model import ObjectAlreadyExistsError +from ultimate_memory.model import ObjectKey +from ultimate_memory.model import ObjectKeyEscapesRootError + + +class MinIOSettings(BaseSettings): + """Connection settings for the self-host S3-compatible object store.""" + + model_config = SettingsConfigDict(env_prefix="UGM_MINIO_", extra="ignore") + + endpoint_url: str + access_key: SecretStr + secret_key: SecretStr + region: str = "us-east-1" + + +class _StreamingBody(Protocol): + """The two response-body operations used by this adapter.""" + + def read(self) -> bytes: + """Read the complete response body.""" + ... + + def close(self) -> None: + """Release the underlying HTTP connection.""" + ... + + +class _GetObjectOutput(TypedDict): + """The fields consumed from an S3 GetObject response.""" + + Body: _StreamingBody + + +class _HeadObjectOutput(TypedDict): + """The fields consumed from an S3 HeadObject response.""" + + Metadata: NotRequired[dict[str, str]] + + +class _ListedObject(TypedDict): + """One object identity returned by ListObjectsV2.""" + + Key: str + + +class _ListObjectsOutput(TypedDict): + """The fields consumed from one ListObjectsV2 response page.""" + + Contents: NotRequired[list[_ListedObject]] + IsTruncated: NotRequired[bool] + NextContinuationToken: NotRequired[str] + + +class _S3Client(Protocol): + """The narrow boto3 client subset the MinIO adapter owns.""" + + def head_bucket(self, *, Bucket: str) -> object: + """Check that one bucket is reachable.""" + ... + + def create_bucket(self, *, Bucket: str) -> object: + """Create one bucket.""" + ... + + def get_object(self, *, Bucket: str, Key: str) -> _GetObjectOutput: + """Read one object.""" + ... + + def put_object( + self, + *, + Bucket: str, + Key: str, + Body: bytes, + IfNoneMatch: str, + Metadata: dict[str, str], + ) -> object: + """Conditionally create one immutable object.""" + ... + + def head_object(self, *, Bucket: str, Key: str) -> _HeadObjectOutput: + """Read one object's metadata.""" + ... + + def delete_object(self, *, Bucket: str, Key: str) -> object: + """Delete one object idempotently.""" + ... + + def list_objects_v2( + self, *, Bucket: str, Prefix: str, ContinuationToken: str = "" + ) -> _ListObjectsOutput: + """List one page beneath a key prefix.""" + ... + + +class MinIOObjectStore: + """Immutable objects in one explicitly selected MinIO bucket.""" + + def __init__( + self, + *, + bucket: str, + settings: MinIOSettings | None = None, + client: _S3Client | None = None, + ) -> None: + """Bind one bucket to either injected test client or configured MinIO.""" + if not bucket: + raise ValueError("a MinIO object store requires a non-empty bucket") + if client is None and settings is None: + raise ValueError("MinIO settings are required when no client is injected") + self._bucket = bucket + self._client = client or _client(settings=cast("MinIOSettings", settings)) + + def ensure_bucket(self) -> None: + """Provision the configured bucket if it does not exist.""" + try: + self._client.head_bucket(Bucket=self._bucket) + return + except ClientError as error: + if _error_code(error=error) not in {"404", "NoSuchBucket", "NotFound"}: + raise + try: + self._client.create_bucket(Bucket=self._bucket) + except ClientError as error: + if _error_code(error=error) not in { + "BucketAlreadyExists", + "BucketAlreadyOwnedByYou", + }: + raise + + def read_bytes(self, *, key: ObjectKey) -> bytes: + """Read all bytes stored at one validated object key.""" + response = self._client.get_object( + Bucket=self._bucket, Key=_validated_key(key=key) + ) + body = response["Body"] + try: + return body.read() + finally: + body.close() + + def write_bytes( + self, *, key: ObjectKey, content: bytes, storage_class: str | None = None + ) -> None: + """Create immutable bytes atomically, refusing an occupied key.""" + metadata = {} if storage_class is None else {"storage-class": storage_class} + try: + self._client.put_object( + Bucket=self._bucket, + Key=_validated_key(key=key), + Body=content, + IfNoneMatch="*", + Metadata=metadata, + ) + except ClientError as error: + if _error_code(error=error) not in { + "409", + "412", + "ConditionalRequestConflict", + "PreconditionFailed", + }: + raise + raise ObjectAlreadyExistsError( + f"object key {key.root!r} is already occupied; objects are immutable" + ) from error + + def storage_class_of(self, *, key: ObjectKey) -> str | None: + """Return the routing class recorded in object metadata, when present.""" + response = self._client.head_object( + Bucket=self._bucket, Key=_validated_key(key=key) + ) + return response.get("Metadata", {}).get("storage-class") + + def purge_objects( + self, *, keys: tuple[ObjectKey, ...], prefixes: tuple[ObjectKey, ...] + ) -> None: + """Idempotently delete exact keys and prefix-boundary descendants.""" + targets = {_validated_key(key=key) for key in keys} + for prefix in prefixes: + targets.update(self._keys_under(prefix=_validated_key(key=prefix))) + for target in sorted(targets): + self._client.delete_object(Bucket=self._bucket, Key=target) + + def verify_objects_purged( + self, *, keys: tuple[ObjectKey, ...], prefixes: tuple[ObjectKey, ...] + ) -> None: + """Fail when any exact key or prefix-boundary descendant remains.""" + remaining: list[str] = [] + for key in keys: + normalized = _validated_key(key=key) + if self._exists(normalized=normalized): + remaining.append(normalized) + for prefix in prefixes: + remaining.extend(self._keys_under(prefix=_validated_key(key=prefix))) + if remaining: + raise RuntimeError( + f"object purge verification found: {sorted(remaining)!r}" + ) + + def _exists(self, *, normalized: str) -> bool: + """Return whether one exact object exists, propagating non-absence errors.""" + try: + self._client.head_object(Bucket=self._bucket, Key=normalized) + return True + except ClientError as error: + if _error_code(error=error) in {"404", "NoSuchKey", "NotFound"}: + return False + raise + + def _keys_under(self, *, prefix: str) -> tuple[str, ...]: + """Enumerate exact and descendant keys without matching sibling prefixes.""" + boundary = prefix.rstrip("/") + result: list[str] = [] + continuation = "" + while True: + page = ( + self._client.list_objects_v2( + Bucket=self._bucket, Prefix=boundary, ContinuationToken=continuation + ) + if continuation + else self._client.list_objects_v2(Bucket=self._bucket, Prefix=boundary) + ) + result.extend( + item["Key"] + for item in page.get("Contents", []) + if item["Key"] == boundary or item["Key"].startswith(f"{boundary}/") + ) + if not page.get("IsTruncated", False): + return tuple(result) + continuation = page.get("NextContinuationToken", "") + if not continuation: + raise RuntimeError( + "MinIO returned a truncated object page without a continuation token" + ) + + +def _client(*, settings: MinIOSettings) -> _S3Client: + """Construct the path-style S3 client supported by local MinIO.""" + return cast( + "_S3Client", + boto3.client( + "s3", + endpoint_url=settings.endpoint_url, + aws_access_key_id=settings.access_key.get_secret_value(), + aws_secret_access_key=settings.secret_key.get_secret_value(), + region_name=settings.region, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ), + ) + + +def _validated_key(*, key: ObjectKey) -> str: + """Reject absolute and parent-traversal keys for local/S3 parity.""" + parts = key.root.split("/") + if key.root.startswith("/") or ".." in parts: + raise ObjectKeyEscapesRootError( + f"object key {key.root!r} escapes the store root" + ) + return key.root + + +def _error_code(*, error: ClientError) -> str: + """Return the provider's stable error code without trimming the exception.""" + return str(error.response.get("Error", {}).get("Code", "")) diff --git a/src/ultimate_memory/profiles/__init__.py b/src/ultimate_memory/profiles/__init__.py index 95af5c3f..735889a9 100644 --- a/src/ultimate_memory/profiles/__init__.py +++ b/src/ultimate_memory/profiles/__init__.py @@ -1 +1,22 @@ """Explicit composition root package.""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ultimate_memory.profiles.selfhost import SelfHostProfile + from ultimate_memory.profiles.selfhost import SelfHostSettings + +__all__ = ("SelfHostProfile", "SelfHostSettings") + + +def __getattr__(name: str) -> object: + """Load a profile only when its explicit composition root is requested.""" + if name == "SelfHostProfile": + from ultimate_memory.profiles.selfhost import SelfHostProfile + + return SelfHostProfile + if name == "SelfHostSettings": + from ultimate_memory.profiles.selfhost import SelfHostSettings + + return SelfHostSettings + raise AttributeError(name) diff --git a/src/ultimate_memory/profiles/selfhost.py b/src/ultimate_memory/profiles/selfhost.py new file mode 100644 index 00000000..e0fffd24 --- /dev/null +++ b/src/ultimate_memory/profiles/selfhost.py @@ -0,0 +1,322 @@ +"""Executable self-host composition for the WP-0.4c Compose quickstart.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys +from typing import Self +from typing import TYPE_CHECKING +from uuid import UUID + +from alembic import command +from alembic.config import Config +from pydantic import Field +from pydantic_settings import BaseSettings +from pydantic_settings import SettingsConfigDict +import sqlalchemy +from sqlalchemy import text +from sqlalchemy.engine import Engine +from sqlalchemy.engine import make_url + +from ultimate_memory.adapters import OpenRouterModelProvider +from ultimate_memory.adapters import OpenRouterSettings +from ultimate_memory.adapters.selfhost import LocalFSForgetManifestStore +from ultimate_memory.adapters.selfhost import MinIOObjectStore +from ultimate_memory.adapters.selfhost import MinIOSettings +from ultimate_memory.model import DeploymentBootstrapInput +from ultimate_memory.model import PipelineStage +from ultimate_memory.spine import DeploymentBootstrapper +from ultimate_memory.spine import RecipeRegistry +from ultimate_memory.spine import seed_canonical_recipes +from ultimate_memory.spine.settings import load_database_settings + +if TYPE_CHECKING: + from fastapi import FastAPI + + from ultimate_memory.adapters.selfhost import SelfHostWorkerLoop + +_SUPPORTED_WORKER_STAGES = (PipelineStage.CONVERT, PipelineStage.STRUCTURE) + + +class SelfHostSettings(BaseSettings): + """One fresh self-host deployment's profile and process settings.""" + + model_config = SettingsConfigDict(env_prefix="UGM_SELFHOST_", extra="ignore") + + deployment_id: UUID + deployment_slug: str = Field(default="local", min_length=1) + deployment_name: str = Field(default="Local memory", min_length=1) + default_language: str = Field(default="en", min_length=1) + raw_bucket_name: str = Field(default="remember-raw", min_length=1) + artifacts_bucket_name: str = Field(default="remember-artifacts", min_length=1) + corpusfs_bucket_name: str = Field(default="remember-corpusfs", min_length=1) + lance_root: Path = Path("/var/lib/ultimate-memory/lance") + forget_manifest_root: Path = Path("/var/lib/ultimate-memory/forget-manifests") + migration_config: Path = Path("alembic.ini") + api_host: str = "0.0.0.0" + api_port: int = Field(default=8000, ge=1, le=65_535) + worker_rate_per_s: float = Field(default=20.0, gt=0) + worker_burst: float = Field(default=20.0, ge=1) + worker_fallback_poll_s: float = Field(default=5.0, gt=0) + worker_session_s: float = Field(default=3_600.0, gt=0) + + +class _FreshDeploymentReadiness: + """Fail closed if a fresh quickstart sees portable forget history. + + WP-0.4c establishes a fresh-deployment Compose skeleton. It must never + silently serve a restored deployment whose D74 manifests require the full + hard-forget recovery composition; finding any manifest stops startup. + """ + + def __init__(self, *, store: LocalFSForgetManifestStore) -> None: + """Bind the separately durable manifest root.""" + self._store = store + + def ensure_ready(self, *, deployment_id: UUID) -> tuple[UUID, ...]: + """Accept an empty root and refuse every non-empty restore.""" + manifests = self._store.manifests(deployment_id=deployment_id) + if manifests: + raise RuntimeError( + "the Compose quickstart found portable hard-forget manifests;" + " restore requires the complete D74 self-host recovery profile" + ) + return () + + +class SelfHostProfile: + """Compose the existing API and E0 workers over PostgreSQL, MinIO, and Lance.""" + + def __init__( + self, + *, + settings: SelfHostSettings, + engine: Engine, + raw_store: MinIOObjectStore, + artifact_store: MinIOObjectStore, + corpusfs_store: MinIOObjectStore, + model_provider: OpenRouterModelProvider, + ) -> None: + """Retain one dependency graph for an API, setup, or worker process.""" + self._settings = settings + self._engine = engine + self._raw_store = raw_store + self._artifact_store = artifact_store + self._corpusfs_store = corpusfs_store + self._model_provider = model_provider + + @classmethod + def from_settings(cls) -> Self: + """Load every external value through its typed settings boundary.""" + profile_settings = SelfHostSettings.model_validate({}) + minio_settings = MinIOSettings.model_validate({}) + return cls( + settings=profile_settings, + engine=sqlalchemy.create_engine( + load_database_settings().sqlalchemy_url(), pool_pre_ping=True + ), + raw_store=MinIOObjectStore( + bucket=profile_settings.raw_bucket_name, settings=minio_settings + ), + artifact_store=MinIOObjectStore( + bucket=profile_settings.artifacts_bucket_name, settings=minio_settings + ), + corpusfs_store=MinIOObjectStore( + bucket=profile_settings.corpusfs_bucket_name, settings=minio_settings + ), + model_provider=OpenRouterModelProvider( + settings=OpenRouterSettings.model_validate({}) + ), + ) + + def close(self) -> None: + """Dispose this process's explicitly owned database pool.""" + self._engine.dispose() + + def setup(self) -> None: + """Apply migrations, provision buckets, bootstrap core rows, and seed recipes.""" + migration = Config(str(self._settings.migration_config)) + migration.set_main_option( + "sqlalchemy.url", load_database_settings().sqlalchemy_url() + ) + command.upgrade(config=migration, revision="head") + self._raw_store.ensure_bucket() + self._artifact_store.ensure_bucket() + self._corpusfs_store.ensure_bucket() + self._settings.forget_manifest_root.mkdir(parents=True, exist_ok=True) + DeploymentBootstrapper(engine=self._engine).bootstrap_deployment( + deployment_input=DeploymentBootstrapInput( + deployment_id=self._settings.deployment_id, + slug=self._settings.deployment_slug, + name=self._settings.deployment_name, + default_language=self._settings.default_language, + raw_bucket=f"s3://{self._settings.raw_bucket_name}", + artifacts_bucket=f"s3://{self._settings.artifacts_bucket_name}", + corpusfs_bucket=f"s3://{self._settings.corpusfs_bucket_name}", + ) + ) + seed_canonical_recipes( + registry=RecipeRegistry(engine=self._engine), + deployment_id=self._settings.deployment_id, + ) + + def api(self) -> FastAPI: + """Build the existing HTTP surface over this self-host dependency graph.""" + from ultimate_memory.adapters.selfhost.lance import LanceChunkIndex + from ultimate_memory.spine import DocumentCatalog + from ultimate_memory.spine import ForgetCatalog + from ultimate_memory.surfaces import build_api + from ultimate_memory.surfaces import QueryEngine + from ultimate_memory.surfaces import RecipeExecutor + from ultimate_memory.surfaces import RecipeSurface + from ultimate_memory.workers.e0 import UploadIngestor + + query_engine = QueryEngine( + engine=self._engine, + search_index=LanceChunkIndex(root=self._settings.lance_root), + model_provider=self._model_provider, + embedding_model="qwen/qwen3-embedding-8b", + ) + app = build_api( + engine=query_engine, + deployment_id=self._settings.deployment_id, + admission=ForgetCatalog(engine=self._engine), + readiness=_FreshDeploymentReadiness( + store=LocalFSForgetManifestStore( + root=self._settings.forget_manifest_root + ) + ), + surface=RecipeSurface( + registry=RecipeRegistry(engine=self._engine), + executor=RecipeExecutor(query_engine=query_engine), + deployment_id=self._settings.deployment_id, + ), + ingest=UploadIngestor( + catalog=DocumentCatalog(engine=self._engine), + raw_store=self._raw_store, + admission=ForgetCatalog(engine=self._engine), + ), + ) + + @app.get("/healthz", include_in_schema=False) + def healthz() -> dict[str, str]: + """Prove the process can reach its authoritative PostgreSQL spine.""" + with self._engine.connect() as connection: + connection.execute(text("SELECT 1")).scalar_one() + return {"status": "ok"} + + return app + + def worker_loop(self, *, stage: PipelineStage) -> SelfHostWorkerLoop: + """Build one E0 route's ordinary LISTEN/NOTIFY worker loop.""" + from ultimate_memory.adapters.selfhost import SelfHostTaskQueue + from ultimate_memory.adapters.selfhost import SelfHostWorkerLoop + from ultimate_memory.adapters.selfhost import TokenBucket + from ultimate_memory.core import ConversionRouter + from ultimate_memory.core import MarkdownPassthroughConverter + from ultimate_memory.model import ProcessingLane + from ultimate_memory.spine import DocumentCatalog + from ultimate_memory.spine import WorkLedger + from ultimate_memory.spine import WorkLedgerSettings + from ultimate_memory.workers import ConvertHandler + from ultimate_memory.workers import HandlerRegistry + from ultimate_memory.workers import StructureHandler + from ultimate_memory.workers import Worker + + if stage not in _SUPPORTED_WORKER_STAGES: + raise ValueError(f"the Compose skeleton has no handler for stage {stage}") + catalog = DocumentCatalog(engine=self._engine) + registry = HandlerRegistry() + if stage is PipelineStage.CONVERT: + registry.register( + stage=stage, + handler=ConvertHandler( + catalog=catalog, + raw_store=self._raw_store, + artifact_store=self._artifact_store, + router=ConversionRouter( + routes={"text/markdown": MarkdownPassthroughConverter()} + ), + ), + ) + else: + registry.register( + stage=stage, + handler=StructureHandler( + catalog=catalog, + artifact_store=self._artifact_store, + model_provider=self._model_provider, + ), + ) + ledger = WorkLedger(engine=self._engine, settings=WorkLedgerSettings()) + return SelfHostWorkerLoop( + worker=Worker( + ledger=ledger, registry=registry, queue=SelfHostTaskQueue(ledger=ledger) + ), + deployment_id=self._settings.deployment_id, + stage=stage, + lane=ProcessingLane.STEADY, + bucket=TokenBucket( + rate_per_s=self._settings.worker_rate_per_s, + capacity=self._settings.worker_burst, + ), + database_url=_psycopg_url(), + fallback_poll_s=self._settings.worker_fallback_poll_s, + ) + + def run_worker(self, *, stage: PipelineStage) -> None: + """Run one configured E0 route until the process is stopped or fails.""" + loop = self.worker_loop(stage=stage) + while True: + loop.run_for(duration_s=self._settings.worker_session_s) + + +def create_api() -> FastAPI: + """Uvicorn factory for the self-host API process.""" + return SelfHostProfile.from_settings().api() + + +def main(argv: list[str] | None = None) -> int: + """Run setup, API, or one E0 worker process for Docker Compose.""" + parser = argparse.ArgumentParser(description="ultimate-memory self-host profile") + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("setup", help="migrate and bootstrap the deployment") + subparsers.add_parser("api", help="serve the deployment HTTP API") + worker = subparsers.add_parser("worker", help="run one E0 worker route") + worker.add_argument( + "--stage", + choices=tuple(stage.value for stage in _SUPPORTED_WORKER_STAGES), + required=True, + ) + args = parser.parse_args(argv) + settings = SelfHostSettings.model_validate({}) + if args.command == "api": + import uvicorn + + uvicorn.run( + create_api(), + host=settings.api_host, + port=settings.api_port, + access_log=True, + ) + return 0 + profile = SelfHostProfile.from_settings() + try: + if args.command == "setup": + profile.setup() + return 0 + profile.run_worker(stage=PipelineStage(args.stage)) + return 0 + finally: + profile.close() + + +def _psycopg_url() -> str: + """Remove SQLAlchemy's driver suffix for psycopg's native connection parser.""" + url = make_url(load_database_settings().sqlalchemy_url()) + return url.set(drivername="postgresql").render_as_string(hide_password=False) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/uv.lock b/uv.lock index d7c687f5..85e83ba6 100644 --- a/uv.lock +++ b/uv.lock @@ -67,6 +67,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924 }, ] +[[package]] +name = "boto3" +version = "1.43.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/da/0e90eab875f2eb4b8708fef2c198f2559ed1e451a1016f1cd4fcdcfbfbe3/boto3-1.43.53.tar.gz", hash = "sha256:c80425acab314d7af09609562053f565139e1fe49108eacfcc1601ebfaee235b", size = 112678 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/27/7e72d25fdde77668b7bd4fa47381192dd2aa64fb77265e4bab786fd9fe2a/boto3-1.43.53-py3-none-any.whl", hash = "sha256:5383e705d8a976a14f23bb8c113c07a396931a019db98fce4cdc68650ec6e4d8", size = 140025 }, +] + +[[package]] +name = "botocore" +version = "1.43.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/6b/ebcefacc4de3cd4f1c449540d86877f76c2f5e586a620831012decbb2b2c/botocore-1.43.53.tar.gz", hash = "sha256:36d93dd8db68ee75f6b61ca9f775161b8168844e4601698701530e6efdded141", size = 15720336 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/e5/1b60e394f0fff97ee70dd16913382b7dffda85b98e639c0c9e8ff56cfaa7/botocore-1.43.53-py3-none-any.whl", hash = "sha256:b7ee9a70d187e5348883c820990ccd9436ab14e2bd6622741fc96fe561e816b8", size = 15404628 }, +] + [[package]] name = "certifi" version = "2026.6.17" @@ -473,6 +501,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419 }, +] + [[package]] name = "ladybug" version = "0.18.2" @@ -1188,6 +1225,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498 }, ] +[[package]] +name = "s3transfer" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/da/4bef7ce7bb989b222aa4785a413896dbec53306dfc59c6ce7d16a7ffbd6a/s3transfer-0.19.1.tar.gz", hash = "sha256:d3d6371dc3f1e5c5427b2b457bcf13bcf87bec334c95aed18642eae61f6926f3", size = 165354 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/23/e84c64ad0e8bc59cd1b2ef98def848deff0ef3456c542afe74d51e9e8c85/s3transfer-0.19.1-py3-none-any.whl", hash = "sha256:d5fd7005ee39307455ad5f310b5ea67f4b1960d7fed5b3671ee50c249de675de", size = 90072 }, +] + [[package]] name = "six" version = "1.17.0" @@ -1315,6 +1364,7 @@ dependencies = [ [package.optional-dependencies] server = [ { name = "alembic" }, + { name = "boto3" }, { name = "fastapi" }, { name = "ladybug" }, { name = "lancedb" }, @@ -1323,11 +1373,13 @@ server = [ { name = "onnxruntime", marker = "python_full_version >= '3.14'" }, { name = "psycopg", extra = ["binary"] }, { name = "sqlalchemy" }, + { name = "uvicorn" }, ] [package.dev-dependencies] dev = [ { name = "alembic" }, + { name = "boto3" }, { name = "fastapi" }, { name = "import-linter" }, { name = "ladybug" }, @@ -1342,11 +1394,13 @@ dev = [ { name = "pytest-cov" }, { name = "ruff" }, { name = "sqlalchemy" }, + { name = "uvicorn" }, ] [package.metadata] requires-dist = [ { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16" }, + { name = "boto3", marker = "extra == 'server'", specifier = ">=1.42" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.139.2" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "ladybug", marker = "extra == 'server'", specifier = ">=0.18.2" }, @@ -1358,11 +1412,13 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.11" }, { name = "pydantic-settings", specifier = ">=2.10" }, { name = "sqlalchemy", marker = "extra == 'server'", specifier = ">=2.0" }, + { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.37" }, ] [package.metadata.requires-dev] dev = [ { name = "alembic", specifier = ">=1.16" }, + { name = "boto3", specifier = ">=1.42" }, { name = "fastapi", specifier = ">=0.139.2" }, { name = "import-linter", specifier = ">=2.13" }, { name = "ladybug", specifier = ">=0.18.2" }, @@ -1377,6 +1433,7 @@ dev = [ { name = "pytest-cov", specifier = ">=6.0" }, { name = "ruff", specifier = ">=0.4" }, { name = "sqlalchemy", specifier = ">=2.0" }, + { name = "uvicorn", specifier = ">=0.37" }, ] [[package]] @@ -1387,3 +1444,16 @@ sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087 }, ] + +[[package]] +name = "uvicorn" +version = "0.51.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/65/b7c6c443ccc58678c91e1e973bbe2a878591538655d6e1d47f24ba1c51f3/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0", size = 94412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/ec/dbb7e5a6b91f86bfb9eb7d2988a2730907b6a729875b949c7f022e8b88fa/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b", size = 73219 }, +] diff --git a/website/src/app/docs/deployment/page.mdx b/website/src/app/docs/deployment/page.mdx new file mode 100644 index 00000000..7198a947 --- /dev/null +++ b/website/src/app/docs/deployment/page.mdx @@ -0,0 +1,85 @@ +export const metadata = { + title: "Self-host Deployment", + description: + "Run the fresh-deployment Docker Compose skeleton with PostgreSQL, MinIO, the API, and E0 workers.", +}; + +# Self-host Deployment + +The repository ships a Docker Compose skeleton for a **fresh, single-deployment** +self-host. It starts PostgreSQL, MinIO, the HTTP API, and separate conversion and +structure workers. The one-shot `setup` service applies every migration before +the API or workers start, provisions the three object buckets, bootstraps the +deployment, and seeds the canonical retrieval recipes. + +This is a pre-release infrastructure proof. It processes a Markdown upload +through conversion and deterministic structure, then leaves its chunk work +pending. Later pipeline workers, the full restore/hard-forget composition, pinned +release images, and the public `remember` names belong to the remaining release +work. Do not expose this stack to untrusted traffic. + +## Start the stack + +Docker Engine with Compose v2 is required. + +```bash +cp .env.example .env +docker compose up --build --detach --wait +``` + +The example file contains local-only PostgreSQL and MinIO credentials. Replace +every secret before using the stack outside an isolated development machine. +It also contains an OpenRouter placeholder so the provider adapter can be +constructed without making a model request. Replace it with a real key before +processing a corpus; the short smoke upload below makes no LLM or embedding call. + +Check the API and the deployment-seeded recipe registry: + +```bash +curl --fail http://localhost:8000/healthz +curl --fail http://localhost:8000/recipes +``` + +## Ingest the smoke document + +Create a short Markdown document and push it through the ordinary E0 API: + +```bash +printf '# Hello\n\nRemember this deployment.\n' > /tmp/remember-smoke.md +curl --fail \ + --data-binary @/tmp/remember-smoke.md \ + 'http://localhost:8000/ingest?filename=remember-smoke.md&mime=text%2Fmarkdown' +``` + +The response names the deployment, document, and version. Both E0 workers use +the normal PostgreSQL work ledger: `convert` and `structure` become `succeeded`, +and the next `chunk` row remains visibly `pending` because this skeleton does not +pretend the later worker topology is composed. + +```bash +docker compose exec postgres psql -U ugm -d ugm -c \ + "SELECT stage, status FROM processing_state ORDER BY created_at" +``` + +The original bytes and derived Markdown/sidecars are in the MinIO buckets. The +MinIO console is available at `http://localhost:9001` with the credentials from +`.env`. + +## Stop and reset + +Stop containers while retaining state: + +```bash +docker compose down +``` + +To erase this local deployment completely, including PostgreSQL, MinIO, Lance, +and the separately mounted forget-manifest directory: + +```bash +docker compose down --volumes +``` + +The second command is destructive. It is appropriate only for this disposable +quickstart deployment—not for backup or restore. Production portability uses +native store tools and preserves the hard-forget manifest root first. diff --git a/website/src/app/docs/project-status/page.mdx b/website/src/app/docs/project-status/page.mdx index 854d898d..d7078922 100644 --- a/website/src/app/docs/project-status/page.mdx +++ b/website/src/app/docs/project-status/page.mdx @@ -22,7 +22,7 @@ Ultimate Memory is being built **design-first**: the complete system — require - **Phase 5, complete** — retrieval complete. The full zero-LLM primitive set is implemented (`fuse`, `rerank`, `transcript`, `delta`, `pages_about`, enumerated `aggregate`, and a streaming batch `scan`, alongside resolve/lookup/search/graph); recipes are registry rows whose declared grain is enforced mechanically; and the self-accounting envelope surfaces contradictions, withdrawn support, mixed-grain parts, identity regime, belief horizons, freshness, and typed negatives. API, CLI, and MCP all render the same deployment registry. The consumption skill is guarded by the repeatable S58 cold-agent eval, while the retrieval spike battery records filtered Lance search at 10 million rows and graph pagination at a 100,000-edge hub. The client-first base wheel now contains the typed SDK, remote CLI and MCP, lineage-aware E0 ingest, and typed remote connector-management commands plus their deployment composition port, without server dependencies; `[server]`, `[connectors-watched-directory]`, and `[k]` name the heavier install surfaces. The public `remember-dev`/`remember` rename remains deliberately gated with release engineering. - **Phase 6, complete** — Plane K now includes the deterministic control plane and crash-safe single-committer driver, exact rule routing and staleness, deterministic fact-sheet and agent-written prose bands, planner/reflection decisions with quarantine and adoption, and authored-page frontmatter, citations, watches, review flags, and debounced workflow dispatch. D73 removed the proposed K3 tier: personal or organizational principles are authored K2 content, supported by compiled scope pages and never machine-promoted or rewritten. - **Phase 3, complete** — the evidence lifecycle: documents that change. Watched sources poll as recorded sync cycles (a local-directory watcher ships; revision and content no-ops, debounce, source-deletion detection); an edited document becomes a new version of its lineage and the full structure route (an LLM-proposed section tree normalized by a deterministic snap) re-reads it; unchanged chunks reuse their prior claims, prefixes, and vectors so cost is proportional to the edit (measured hit rate 0.79 on the spike corpus); reconciliation transitions testimony currency on an append-only ledger, recounts evidence by distinct current lineages, closes solely-supported facts when the source withdrew them (at the sync-cycle barrier, so a moved section is a support swap — never a retract flicker) and flags them for review when only the toolchain changed; deletion removes a document's contribution uniformly while keeping its claims as history; and a standing `lifecycle` eval suite guards the cache/ledger/count invariants with planted regression canaries. Those lifecycle events now feed the graph, corpus filesystem, and Plane-K control plane described above. -- **Phase 7, in progress** — operational correctness now includes resumable initial-load and version-bump backfill on the same work ledger, a reproducible PostgreSQL scale battery for the designed partitions, indexes, hubs, and batching invariants, and optional per-deployment/stage/lane cost ceilings. Budget exhaustion parks healthy work durably without consuming an attempt; the local CLI reports current spend, remaining budget, tier attribution, and parked work from the same authoritative rows. Typed worker telemetry preserves complete exception chains, while bounded CLI inspection exposes pipeline/DLQ state, poison targets, P2/P3 pointers, and currency-ledger drift; an explicit one-row replay grants only additional attempts, and rebuild drills call the production P2/P3 builders. Hard-forget now purges every active library-controlled surface and replays a separately durable, content-free manifest before serving after a restore. Portability is deliberately smaller than a backup product: operators move PostgreSQL, objects, and Git with native tools; the library defines the fail-closed restore order and rebuilds P1/P2/P3. Timings remain measurements rather than hosted SLAs, and no export/import CLI, dashboard, billing policy, HA topology, backup scheduler, or control plane enters the library. +- **Phase 7, in progress** — operational correctness now includes resumable initial-load and version-bump backfill on the same work ledger, a reproducible PostgreSQL scale battery for the designed partitions, indexes, hubs, and batching invariants, and optional per-deployment/stage/lane cost ceilings. Budget exhaustion parks healthy work durably without consuming an attempt; the local CLI reports current spend, remaining budget, tier attribution, and parked work from the same authoritative rows. Typed worker telemetry preserves complete exception chains, while bounded CLI inspection exposes pipeline/DLQ state, poison targets, P2/P3 pointers, and currency-ledger drift; an explicit one-row replay grants only additional attempts, and rebuild drills call the production P2/P3 builders. Hard-forget now purges every active library-controlled surface and replays a separately durable, content-free manifest before serving after a restore. Portability is deliberately smaller than a backup product: operators move PostgreSQL, objects, and Git with native tools; the library defines the fail-closed restore order and rebuilds P1/P2/P3. A fresh-deployment Compose skeleton now wires PostgreSQL, MinIO, the API, and the conversion/structure workers through the real profile; it is an infrastructure proof rather than a published or production-ready deployment. Timings remain measurements rather than hosted SLAs, and no export/import CLI, dashboard, billing policy, HA topology, backup scheduler, or control plane enters the library. ## The build order diff --git a/website/src/lib/docs/navigation.ts b/website/src/lib/docs/navigation.ts index fda7ecec..a678bb3a 100644 --- a/website/src/lib/docs/navigation.ts +++ b/website/src/lib/docs/navigation.ts @@ -10,6 +10,7 @@ export const docsNavigation: NavItem[] = [ { title: "Introduction", href: "/docs" }, { title: "Concepts", href: "/docs/concepts" }, { title: "Architecture", href: "/docs/architecture" }, + { title: "Self-host Deployment", href: "/docs/deployment" }, { title: "Mounts and Skill", href: "/docs/mounts" }, { title: "Project Status", href: "/docs/project-status" }, {