feat: add Compose self-host quickstart

.dockerignore

@@ -0,0 +1,15 @@
+.git
+.github
+.pytest_cache
+.venv
+.coverage*
+**/__pycache__
+**/*.pyc
+dist
+htmlcov
+plan
+src/tests
+website
+_prompts
+.loopy_loop
+.eval-banana

.env.example

@@ -0,0 +1,16 @@
+# Local-only quickstart credentials. Replace every secret before exposing the stack.
+UGM_POSTGRES_USER=ugm
+UGM_POSTGRES_PASSWORD=ugm-local-only
+UGM_POSTGRES_DB=ugm
+UGM_MINIO_ACCESS_KEY=remember-local
+UGM_MINIO_SECRET_KEY=remember-local-only
+
+# One stable deployment identity; changing it creates a different trust domain.
+UGM_SELFHOST_DEPLOYMENT_ID=11111111-1111-4111-8111-111111111111
+UGM_SELFHOST_DEPLOYMENT_SLUG=local
+UGM_SELFHOST_DEPLOYMENT_NAME=Local memory
+UGM_SELFHOST_API_PORT=8000
+
+# Required to start the provider adapter. The short Markdown smoke path makes no
+# provider request; replace this value before processing a real corpus.
+UGM_OPENROUTER_API_KEY=replace-before-real-use

.github/workflows/ci.yml

@@ -10,6 +10,66 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  compose-quickstart:
+    name: Compose quickstart
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate Compose model
+        run: docker compose --env-file .env.example config --quiet
+
+      - name: Start fresh self-host deployment
+        run: docker compose --env-file .env.example up --build --detach --wait
+
+      - name: Prove MinIO conditional create
+        run: |
+          docker compose --env-file .env.example exec -T api python - <<'PY'
+          from ultimate_memory.adapters.selfhost import MinIOObjectStore, MinIOSettings
+          from ultimate_memory.model import ObjectAlreadyExistsError, ObjectKey
+
+          store = MinIOObjectStore(
+              bucket="remember-raw", settings=MinIOSettings.model_validate({})
+          )
+          key = ObjectKey("ci/immutable-object")
+          store.write_bytes(key=key, content=b"first")
+          try:
+              store.write_bytes(key=key, content=b"replacement")
+          except ObjectAlreadyExistsError:
+              pass
+          else:
+              raise RuntimeError("MinIO replaced an immutable object key")
+          PY
+
+      - name: Ingest and observe the E0 chain
+        run: |
+          curl --fail --data-binary $'# Hello\n\nCompose smoke.\n' \
+            'http://localhost:8000/ingest?filename=smoke.md&mime=text%2Fmarkdown'
+          observed=''
+          for attempt in {1..30}; do
+            observed="$(docker compose --env-file .env.example exec -T postgres \
+              psql -U ugm -d ugm -Atc \
+              "SELECT count(*) FILTER (WHERE stage='convert' AND status='succeeded'), count(*) FILTER (WHERE stage='structure' AND status='succeeded'), count(*) FILTER (WHERE stage='chunk' AND status='pending') FROM processing_state")"
+            if [ "$observed" = '1|1|1' ]; then
+              break
+            fi
+            sleep 1
+          done
+          test "$observed" = '1|1|1'
+          test "$(docker compose --env-file .env.example exec -T postgres \
+            psql -U ugm -d ugm -Atc 'SELECT count(*) FROM cost_ledger')" = '0'
+
+      - name: Show service logs after failure
+        if: failure()
+        run: docker compose --env-file .env.example logs --no-color
+
+      - name: Remove quickstart deployment
+        if: always()
+        run: docker compose --env-file .env.example down --volumes
+
   checks:
     runs-on: ubuntu-latest
     permissions:

.gitignore

@@ -1,6 +1,7 @@
 .idea/
 _additional_context/
 _feature_planning/
+.env
 
 # python
 .venv/

Dockerfile

@@ -0,0 +1,28 @@
+FROM ghcr.io/astral-sh/uv:0.11.31@sha256:ecd4de2f060c64bea0ff8ecb182ddf46ba3fcccdc8a60cfdbaf20d1a047d7437 AS uv
+
+FROM python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6
+
+COPY --from=uv /uv /uvx /bin/
+
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PYTHONUNBUFFERED=1 \
+    UV_LINK_MODE=copy
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock README.md LICENSE alembic.ini ./
+
+RUN addgroup --system app \
+    && adduser --system --ingroup app app \
+    && mkdir -p /var/lib/ultimate-memory/forget-manifests \
+    && chown -R app:app /var/lib/ultimate-memory \
+    && uv sync --locked --no-dev --extra server --no-install-project
+
+COPY src ./src
+
+RUN uv sync --locked --no-dev --extra server
+
+USER app
+
+ENTRYPOINT ["python", "-m", "ultimate_memory.profiles.selfhost"]
+CMD ["api"]

README.md

@@ -8,10 +8,11 @@ distill them into progressively more abstract, navigable knowledge — while kee
 auditable by humans. Scale is a requirement, not an aspiration: it is meant to still be useful
 at a million documents.
 
-> **⚠️ Implementation has just begun; there is nothing usable yet.** The design came first and
-> lives here in full — requirements, architecture, research, decisions, and the open questions.
-> The build follows [plan/plans/roadmap.md](plan/plans/roadmap.md) (phase 0 is under way). If
-> you're looking for a working library, it isn't here yet.
+> **Pre-release software.** Phases 0–6 and most of Phase 7 are implemented and tested, but no
+> public package or container release exists yet. The fresh-deployment Docker Compose skeleton
+> is documented under [Self-host deployment](website/src/app/docs/deployment/page.mdx); it proves
+> PostgreSQL, MinIO, API ingestion, and the first two E0 worker stages, not a production rollout.
+> The build follows [plan/plans/roadmap.md](plan/plans/roadmap.md).
 
 ## TL;DR
 

compose.yaml

@@ -0,0 +1,98 @@
+name: ultimate-memory
+
+x-app: &app
+  build:
+    context: .
+    dockerfile: Dockerfile
+  image: ultimate-memory:local
+  environment:
+    UGM_DATABASE_URL: postgresql+psycopg://${UGM_POSTGRES_USER}:${UGM_POSTGRES_PASSWORD}@postgres:5432/${UGM_POSTGRES_DB}
+    UGM_MINIO_ENDPOINT_URL: http://minio:9000
+    UGM_MINIO_ACCESS_KEY: ${UGM_MINIO_ACCESS_KEY}
+    UGM_MINIO_SECRET_KEY: ${UGM_MINIO_SECRET_KEY}
+    UGM_OPENROUTER_API_KEY: ${UGM_OPENROUTER_API_KEY}
+    UGM_SELFHOST_DEPLOYMENT_ID: ${UGM_SELFHOST_DEPLOYMENT_ID}
+    UGM_SELFHOST_DEPLOYMENT_SLUG: ${UGM_SELFHOST_DEPLOYMENT_SLUG}
+    UGM_SELFHOST_DEPLOYMENT_NAME: ${UGM_SELFHOST_DEPLOYMENT_NAME}
+    UGM_SELFHOST_API_PORT: "8000"
+  volumes:
+    - app-state:/var/lib/ultimate-memory
+    - forget-manifests:/var/lib/ultimate-memory/forget-manifests
+
+services:
+  postgres:
+    image: ghcr.io/dbsystel/postgresql-partman@sha256:bf9d2331aaeb3e33a4e6e73b4c024ca618d4d1ea050bbab978125ff63b83385a
+    environment:
+      POSTGRES_USER: ${UGM_POSTGRES_USER}
+      POSTGRES_PASSWORD: ${UGM_POSTGRES_PASSWORD}
+      POSTGRES_DB: ${UGM_POSTGRES_DB}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${UGM_POSTGRES_USER} -d ${UGM_POSTGRES_DB}"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+  minio:
+    image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
+    command: server /data --console-address :9001
+    environment:
+      MINIO_ROOT_USER: ${UGM_MINIO_ACCESS_KEY}
+      MINIO_ROOT_PASSWORD: ${UGM_MINIO_SECRET_KEY}
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    ports:
+      - "9001:9001"
+    volumes:
+      - minio-data:/data
+
+  setup:
+    <<: *app
+    command: ["setup"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+
+  api:
+    <<: *app
+    command: ["api"]
+    depends_on:
+      setup:
+        condition: service_completed_successfully
+    healthcheck:
+      test:
+        - CMD
+        - python
+        - -c
+        - "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz', timeout=2).read()"
+      interval: 10s
+      timeout: 5s
+      retries: 30
+    ports:
+      - "${UGM_SELFHOST_API_PORT}:8000"
+
+  worker-convert:
+    <<: *app
+    command: ["worker", "--stage", "convert"]
+    depends_on:
+      setup:
+        condition: service_completed_successfully
+
+  worker-structure:
+    <<: *app
+    command: ["worker", "--stage", "structure"]
+    depends_on:
+      setup:
+        condition: service_completed_successfully
+
+volumes:
+  app-state:
+  forget-manifests:
+  minio-data:
+  postgres-data:

plan/plans/phase-0-foundations.md

@@ -24,7 +24,7 @@ scaffold exists with ≥1 seeded doc.
 | WP-0.4 | **The D61 port interfaces** (`ports/` Protocols: object store, task queue, mounts, git remote, model provider, telemetry, auth) + import-linter contracts in CI | packaging §3–4; D61, D62 | WP-0.1 | `ports/` + CI architecture checks | illegal import fails CI (proven by a deliberate violation) | done |
 | WP-0.4a | **Self-host adapters**: pg-queue delivery shell (`LISTEN/NOTIFY` + `SKIP LOCKED`, transactional enqueue, token-bucket rate limits), local-FS object store, local mount publisher, `adapters/testing` tier (MinIO wiring lands with WP-0.4c) | packaging §3, §5; D62 | WP-0.4, WP-0.3 | `adapters/selfhost` + `adapters/testing` | demo chain runs against real Postgres with zero GCP deps; transactional-enqueue crash test | done |
 | WP-0.4b | **Reference adapters**: Cloud Tasks push shell + dispatch server, GCS store, gcsfuse publisher; **the janitor sweep** (shared, port-agnostic) | packaging §3; orchestration §2–3; D61 | WP-0.4 | `adapters/gcp` + janitor job | same demo chain on the GCP profile; janitor re-announces a killed delivery on BOTH profiles | planned |
-| WP-0.4c | **Compose self-host profile** (postgres + minio + api + worker; `profiles/selfhost`) — the quickstart skeleton | packaging §5 | WP-0.4a | docker-compose + profile module | `docker compose up` → demo ingest → state rows; CI-run | planned |
+| WP-0.4c | **Compose self-host profile** (postgres + minio + api + worker; `profiles/selfhost`) — the quickstart skeleton | packaging §5 | WP-0.4a | docker-compose + profile module | `docker compose up` → demo ingest → state rows; CI-run | done |
 | WP-0.5 | **Eval harness skeleton** (questions #14 — this WP owns it): golden-set storage (`golden_pairs`, `golden_claim_labels`, `canary_cases`, `eval_runs`), suite runner, CI wiring | schema §5; D22; registries §10 | WP-0.2 | harness package + `eval` CI job | empty suites run; a seeded canary fails deliberately and blocks CI | done |
 | WP-0.6 | Golden-set labeling tooling (LLM-propose / human-adjudicate loop, circularity guard) | D22; registries §11.1 | WP-0.5 | labeling CLI | 20 seed pairs labeled end-to-end | planned |
 | WP-0.7 | Blockizer golden corpus scaffold (expected block-hash regression per `blockizer_version`) | e1 §2 (D57) | WP-0.5 | corpus + CI check | seeded doc's hashes locked; a deliberate parser change trips CI | done |
@@ -99,6 +99,17 @@ enqueue delivers exactly its row's wake — the schema trigger's by-construction
 announce re-delivery, and the demo chain draining through the loop with zero GCP dependencies.
 MinIO wiring lands with WP-0.4c per the sequencing amendment.
 
+**WP-0.4c complete (2026-07-22):** the fresh-deployment Compose skeleton wires the
+real `profiles/selfhost` composition to pinned PostgreSQL 16 + pg_partman and MinIO images,
+one migration/bootstrap service, the HTTP API, and separate `convert`/`structure` workers.
+The MinIO adapter preserves write-once object semantics, path boundaries, routing metadata,
+and hard-forget purge verification. A real Compose acceptance started from empty named volumes,
+served health + recipes, ingested Markdown through MinIO, and observed `convert=succeeded`,
+`structure=succeeded`, and `chunk=pending` in the authoritative work ledger with zero cost rows
+(no provider call). CI repeats that proof in a job parallel to the Python matrix. This remains a
+fresh, pre-release infrastructure skeleton: later-stage workers, authentication, restore
+recovery, published GHCR images, and the public rename stay in their owning release work.
+
 
 **WP-0.5 + WP-0.7 complete (2026-07-18):** the eval-harness skeleton lands in `eval/harness.py`
 (suites over `canary_cases`, runs recorded in `eval_runs`, per-suite evaluator registration;
@@ -116,7 +127,7 @@ the edited block, offsets slice document.md exactly.
 cleanly (WP-0.2); the no-op worker runs end-to-end through the task-queue delivery port with
 `processing_state` + `cost_ledger` rows (WP-0.3 + WP-0.4a, per the sequencing amendment); the
 eval harness runs empty suites in CI (WP-0.5); the blockizer golden corpus exists with a seeded
-doc (WP-0.7). Carried forward, not exit-blocking: WP-0.4b/0.4c (GCP parity + compose,
+doc (WP-0.7). Carried forward, not exit-blocking: WP-0.4b (GCP parity,
 Phase-1-parallel) and WP-0.6 (golden-set labeling tooling, needed by Phase 2's measured
 thresholds).
 

plan/plans/phase-5-retrieval-complete.md

@@ -16,4 +16,4 @@ skill; latency budget measured against the §10 envelope.
 | WP-5.4 | Surfaces: MCP (rendered from registry), CLI parity, API auth perimeter hooks | retrieval §7; D50–D51 | WP-5.2 | MCP server + CLI | tool list = registry; parity test | done (PR #108; `RecipeSurface`/`RecipeMcpServer` + API `/recipes`+`/recipe/{name}`+auth + `ugm query` CLI; parity test + reference docs) |
 | WP-5.5 | **Consumption skill v1** (per-deployment rendered) + the S58 protocol as a repeatable eval | retrieval §8; D51 | WP-5.3 | skill + S58 harness | S58 green with a cold harness | done (PR #109; deployment/scopes/K/mount/recipe-rendered `SKILL.md`; provider-backed S58 retrieval canary) |
 | WP-5.6 | Retrieval spikes: Lance scale, hub pagination, rerank weights, envelope overhead, hydration batching, resolve context | retrieval §13 | WP-5.3 | spike reports + tuned constants | recorded in eval_runs | done (PR #110; six-spike battery recorded at 10M Lance rows and a 100k-edge hub; measured retrieval defaults and regressions implemented) |
-| WP-5.7 | **PyPI packaging of the client surface**: base install = SDK + CLI + MCP; extras `[server]`/`[connectors-*]`/`[k]`; **lineage-aware ingest** in SDK/CLI; connector-management commands | packaging §1–2; D62 | WP-5.4 | the pip package (dist `remember-dev` — decided, questions.md §11a) | fresh-venv install → query + ingest against a compose deployment; push-feeder lineage test (stable source_ref → versions) | done (PR #111; client-first wheel + SDK/CLI/remote MCP, lineage-aware E0 ingest, and connector-management contracts; exact compose replay remains coupled to carried WP-0.4c) |
+| WP-5.7 | **PyPI packaging of the client surface**: base install = SDK + CLI + MCP; extras `[server]`/`[connectors-*]`/`[k]`; **lineage-aware ingest** in SDK/CLI; connector-management commands | packaging §1–2; D62 | WP-5.4 | the pip package (dist `remember-dev` — decided, questions.md §11a) | fresh-venv install → query + ingest against a compose deployment; push-feeder lineage test (stable source_ref → versions) | done (PR #111; client-first wheel + SDK/CLI/remote MCP, lineage-aware E0 ingest, and connector-management contracts; the Compose deployment proof is now supplied by completed WP-0.4c) |

plan/plans/roadmap.md

@@ -95,7 +95,7 @@ needs to restate it:
 
 | Phase | Name | Builds | Keyed to designs | Status |
 |---|---|---|---|---|
-| 0 | Foundations + harness | scaffolding, migrations, tenancy, `processing_state`/`cost_ledger`, queues, **eval harness + golden-set tooling**, blockizer golden corpus | schema; orchestration §1–2; D22 | done (exit criteria met 2026-07-18; WP-0.4b/0.4c + WP-0.6 carried per the phase file) |
+| 0 | Foundations + harness | scaffolding, migrations, tenancy, `processing_state`/`cost_ledger`, queues, **eval harness + golden-set tooling**, blockizer golden corpus | schema; orchestration §1–2; D22 | done (exit criteria met 2026-07-18; WP-0.4c completed 2026-07-22; WP-0.4b + WP-0.6 remain carried per the phase file) |
 | 1 | Walking skeleton | one document end-to-end, everything minimal; 4 retrieval primitives + envelope core | e0, e1, e2_e3, observations, retrieval §2–3 | done (exit criteria met 2026-07-18; PRs #81-#87 — see the phase file) |
 | 2 | Truth machinery | full ER cascade + registries + review queue; supersession; observation adjudication; thresholds measured | registries, e2_e3 §5, observations | done (exit criteria met 2026-07-19; PRs #88-#94 — see the phase file) |
 | 3 | Evidence lifecycle | lineages/versions, Drive connector + sync cycles, currency + counting + reconciliation, chunk reuse, deletion grains, full PageIndex route | evidence_lifecycle, e1 §7, e0 | done (exit criteria met 2026-07-19; PRs #95-#99 — see the phase file) |

pyproject.toml

@@ -42,6 +42,7 @@ dependencies = [
 [project.optional-dependencies]
 server = [
     "alembic>=1.16",
+    "boto3>=1.42",
     "fastapi>=0.139.2",
     "ladybug>=0.18.2",
     "lancedb>=0.34.0",
@@ -50,6 +51,7 @@ server = [
     "onnxruntime>=1.24.1; python_version >= '3.14'",
     "psycopg[binary]>=3.2",
     "sqlalchemy>=2.0",
+    "uvicorn>=0.37",
 ]
 # The watched-directory connector uses only the standard library. Other
 # connector extras are added with their adapters, rather than speculatively.
@@ -68,6 +70,7 @@ Issues = "https://github.com/writeitai/ultimate-memory/issues"
 [dependency-groups]
 dev = [
     "alembic>=1.16",
+    "boto3>=1.42",
     "fastapi>=0.139.2",
     "import-linter>=2.13",
     "ladybug>=0.18.2",
@@ -82,6 +85,7 @@ dev = [
     "pytest-cov>=6.0",
     "ruff>=0.4",
     "sqlalchemy>=2.0",
+    "uvicorn>=0.37",
 ]
 
 [tool.hatch.build.targets.wheel]