From f1022cd31899d85f3c90bdf78e40237564dbda37 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 10:07:04 +0200 Subject: [PATCH 01/12] feat(db/board-service): Postgres persistence Co-Authored-By: Claude Fable 5 --- apps/api/forge_api/routers/board.py | 52 +- apps/api/forge_api/settings.py | 6 + packages/board-core/forge_board/__init__.py | 2 + .../board-core/forge_board/sql_service.py | 699 ++++++++++++++++++ .../tests/test_sql_board_service.py | 602 +++++++++++++++ packages/db/forge_db/models/__init__.py | 2 + packages/db/forge_db/models/planning.py | 42 ++ .../db/migrations/versions/0001_baseline.py | 3 + .../versions/0024_board_persistence.py | 79 ++ packages/db/tests/test_models.py | 3 + 10 files changed, 1475 insertions(+), 15 deletions(-) create mode 100644 packages/board-core/forge_board/sql_service.py create mode 100644 packages/board-core/tests/test_sql_board_service.py create mode 100644 packages/db/migrations/versions/0024_board_persistence.py diff --git a/apps/api/forge_api/routers/board.py b/apps/api/forge_api/routers/board.py index 41406b31..35b65e96 100644 --- a/apps/api/forge_api/routers/board.py +++ b/apps/api/forge_api/routers/board.py @@ -20,11 +20,14 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status from pydantic import BaseModel +from sqlalchemy.orm import Session, sessionmaker from forge_api.auth.rbac import Permission +from forge_api.db import get_session_factory from forge_api.deps import Principal, get_current_principal from forge_api.routers._rbac import require_permission -from forge_board import InMemoryBoardService +from forge_api.settings import get_settings +from forge_board import InMemoryBoardService, SqlAlchemyBoardService from forge_board.exceptions import ( CycleError, EntityNotFoundError, @@ -32,6 +35,7 @@ ) from forge_contracts import ( BoardFilter, + BoardService, BulkUpdate, EpicDTO, IncidentDTO, @@ -64,34 +68,52 @@ class BoardServiceRegistry: - """Vends a :class:`InMemoryBoardService` per workspace (tenant isolation). + """Vends a :class:`~forge_contracts.BoardService` per workspace (tenant isolation). Phase-2 bug fix r4: the board is the primary write surface and must enforce the spec's mandatory per-workspace isolation (design doc section 4; plan Task 1.15) like every other tenant surface hardened in r3. The frozen ``BoardService`` protocol has no ``workspace_id`` dimension, and the entities (``TaskDTO`` etc.) carry no workspace field, so isolation is achieved by - giving each workspace its **own** in-memory service instance. A caller can - therefore only ever see, fetch, mutate, or delete entities in its own - workspace; a foreign id simply does not exist there (404), and listing never - spans tenants. The DB-backed service swapped in at the wire-up barrier filters - by ``workspace_id`` behind the same dependency. + giving each workspace its **own** service instance. A caller can therefore + only ever see, fetch, mutate, or delete entities in its own workspace; a + foreign id simply does not exist there (404), and listing never spans tenants. + + The backend is chosen once (``FORGE_BOARD_BACKEND``): ``memory`` (default) + vends a hermetic :class:`InMemoryBoardService`; ``db`` vends a + :class:`SqlAlchemyBoardService` bound to the workspace + shared session + factory, which filters by ``workspace_id`` behind the same dependency. Both + satisfy the same frozen ``BoardService`` protocol, so the router is agnostic. """ - def __init__(self) -> None: - self._services: dict[uuid.UUID, InMemoryBoardService] = {} - - def for_workspace(self, workspace_id: uuid.UUID) -> InMemoryBoardService: + def __init__( + self, + *, + backend: str = "memory", + session_factory: sessionmaker[Session] | None = None, + ) -> None: + self._backend = backend + self._session_factory = session_factory + self._services: dict[uuid.UUID, BoardService] = {} + + def for_workspace(self, workspace_id: uuid.UUID) -> BoardService: service = self._services.get(workspace_id) if service is None: - service = InMemoryBoardService() + if self._backend == "db": + if self._session_factory is None: # pragma: no cover - misconfiguration + raise RuntimeError("db board backend requires a session factory") + service = SqlAlchemyBoardService(self._session_factory, workspace_id) + else: + service = InMemoryBoardService() self._services[workspace_id] = service return service @lru_cache(maxsize=1) def _board_registry_singleton() -> BoardServiceRegistry: - return BoardServiceRegistry() + if get_settings().board_backend == "db": + return BoardServiceRegistry(backend="db", session_factory=get_session_factory()) + return BoardServiceRegistry(backend="memory") def get_board_registry() -> BoardServiceRegistry: @@ -102,7 +124,7 @@ def get_board_registry() -> BoardServiceRegistry: def get_board_service( principal: Annotated[Principal, Depends(get_current_principal)], registry: Annotated[BoardServiceRegistry, Depends(get_board_registry)], -) -> InMemoryBoardService: +) -> BoardService: """Return the board service scoped to the caller's workspace. Scoping happens here (not in the handlers) so every route — reads and writes @@ -112,7 +134,7 @@ def get_board_service( return registry.for_workspace(principal.workspace_id) -BoardServiceDep = Annotated[InMemoryBoardService, Depends(get_board_service)] +BoardServiceDep = Annotated[BoardService, Depends(get_board_service)] # --------------------------------------------------------------------------- # diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index 3cd41c77..50575cc7 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -106,6 +106,12 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: database_url: str = DEFAULT_DATABASE_URL redis_url: str = DEFAULT_REDIS_URL + # F01 board backend selection. ``memory`` (default) keeps the hermetic, + # process-memory ``InMemoryBoardService`` (unit-test default, no Postgres); + # ``db`` wires the Postgres-backed ``SqlAlchemyBoardService`` behind the same + # frozen ``BoardService`` protocol. Read via ``FORGE_BOARD_BACKEND``. + board_backend: str = "memory" + # Filesystem root for the spec engine's SDD artifacts (manifests, plans). spec_root: str = "specs" diff --git a/packages/board-core/forge_board/__init__.py b/packages/board-core/forge_board/__init__.py index 2f238b27..2c6bcb61 100644 --- a/packages/board-core/forge_board/__init__.py +++ b/packages/board-core/forge_board/__init__.py @@ -23,6 +23,7 @@ ) from forge_board.service import InMemoryBoardService from forge_board.sprint_state import SprintStateMachine +from forge_board.sql_service import SqlAlchemyBoardService from forge_board.velocity import ( BurndownPoint, ScopeEvent, @@ -51,6 +52,7 @@ "SprintStateMachine", "SprintTaskSnapshot", "SprintWindow", + "SqlAlchemyBoardService", "VelocityResult", "VelocitySummary", "__version__", diff --git a/packages/board-core/forge_board/sql_service.py b/packages/board-core/forge_board/sql_service.py new file mode 100644 index 00000000..79844a38 --- /dev/null +++ b/packages/board-core/forge_board/sql_service.py @@ -0,0 +1,699 @@ +"""Postgres-backed :class:`~forge_contracts.BoardService` (F01 board persistence). + +A drop-in, workspace-scoped alternative to :class:`InMemoryBoardService` that +persists the *same* board DTOs (Epic / Task / Sprint / Milestone / Incident, the +dependency graph, the status workflow, bulk ops, and saved-filter queries) to +real Postgres via ``forge_db``. It implements the same frozen ``BoardService`` +protocol, so the API/service factory swaps it in behind ``FORGE_BOARD_BACKEND=db`` +with no behavioural change — the default stays ``memory`` and the in-memory store +remains the unit-test default. + +Scoping mirrors the router's per-workspace ``BoardServiceRegistry``: one instance +is bound to a single ``workspace_id`` and only ever sees/mutates entities in that +workspace (a foreign id is a 404 — ``EntityNotFoundError`` — never a leak). The +``BoardService`` protocol carries no ``workspace_id`` dimension, so it is supplied +at construction, exactly as the registry vends a per-workspace in-memory service. + +Filtering + pagination reuse the in-memory store's own predicates over freshly +rebuilt DTOs, so saved-filter semantics are byte-identical across both backends. + +Fidelity notes vs the in-memory store (both satisfy the same protocol): + +* ``key`` keeps the ``TASK-N`` / ``EPIC-N`` / ``INC-N`` shape, allocated as + ``max(existing suffix) + 1`` within the workspace (the ``(workspace_id, key)`` + unique constraint guards collisions). +* ``knowledge_scope`` / ``handoff_rules`` round-trip ``None`` faithfully: a DTO + ``None`` is stored as JSON ``null`` and read back as ``None``, distinct from an + all-defaults empty object. +* Referential integrity is real: a task's ``project_id`` must reference an + existing project and a dependency edge must reference existing tasks. This is + the one storage-boundary divergence from the hermetic in-memory store (which + holds free-floating DTOs); ``start_date`` / ``end_date`` / ``due_date`` land in + the pre-existing naive ``timestamp`` columns, so aware inputs are normalised to + naive UTC. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from sqlalchemy import delete, select + +from forge_board.exceptions import EntityNotFoundError +from forge_board.graph import has_cycle, would_create_cycle +from forge_board.service import ( + _epic_matches, + _paginate, + _simple_matches, + _task_matches, +) +from forge_board.workflow import validate_transition +from forge_contracts import ( + AcceptanceCriterion, + ApprovalPolicy, + BoardFilter, + BulkUpdate, + CycleError, + EpicDTO, + HandoffRules, + IncidentDTO, + KnowledgeScope, + MilestoneDTO, + RepoTarget, + SprintDTO, + SubAgentPolicy, + TaskDTO, + TaskStatus, +) +from forge_contracts import enums as ce +from forge_db.models import Epic, Incident, Milestone, Sprint, Task, TaskDependency +from forge_db.models import enums as dbe + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _ev(value: object) -> str: + """Return an enum member's ``.value`` (or the value itself if already a str).""" + return value.value if hasattr(value, "value") else str(value) + + +def _naive(dt: datetime | None) -> datetime | None: + """Normalise an (optionally aware) datetime to naive UTC for a naive column.""" + if dt is None: + return None + if dt.tzinfo is not None: + return dt.astimezone(UTC).replace(tzinfo=None) + return dt + + +def _dedup(ids: list[uuid.UUID]) -> list[uuid.UUID]: + """Order-preserving de-duplication (the adjacency table is a set of edges).""" + seen: set[uuid.UUID] = set() + out: list[uuid.UUID] = [] + for i in ids: + if i not in seen: + seen.add(i) + out.append(i) + return out + + +class SqlAlchemyBoardService: + """A Postgres-backed board domain service (implements ``BoardService``).""" + + def __init__(self, session_factory: sessionmaker[Session], workspace_id: uuid.UUID) -> None: + self._sf = session_factory + self._ws = workspace_id + + # ------------------------------------------------------------------ # + # Shared internals # + # ------------------------------------------------------------------ # + + def _get(self, session: Session, model: type, entity_id: uuid.UUID, entity: str): + row = session.get(model, entity_id) + if row is None or row.workspace_id != self._ws: + raise EntityNotFoundError(entity, entity_id) + return row + + def _next_key(self, session: Session, model: type, prefix: str) -> str: + keys = ( + session.execute( + select(model.key).where( + model.workspace_id == self._ws, model.key.like(f"{prefix}-%") + ) + ) + .scalars() + .all() + ) + max_n = 0 + for key in keys: + suffix = str(key).rsplit("-", 1)[-1] + if suffix.isdigit(): + max_n = max(max_n, int(suffix)) + return f"{prefix}-{max_n + 1}" + + def _task_edges(self, session: Session) -> dict[uuid.UUID, set[uuid.UUID]]: + edges: dict[uuid.UUID, set[uuid.UUID]] = {} + for row in ( + session.execute( + select(TaskDependency).where(TaskDependency.workspace_id == self._ws) + ) + .scalars() + .all() + ): + edges.setdefault(row.task_id, set()).add(row.depends_on_id) + return edges + + def _depends_on(self, session: Session, task_id: uuid.UUID) -> list[uuid.UUID]: + rows = ( + session.execute( + select(TaskDependency).where( + TaskDependency.workspace_id == self._ws, + TaskDependency.task_id == task_id, + ) + ) + .scalars() + .all() + ) + rows.sort(key=lambda r: (r.created_at, r.id)) + return [r.depends_on_id for r in rows] + + def _set_edges( + self, session: Session, task_id: uuid.UUID, depends_on: list[uuid.UUID] + ) -> None: + session.execute( + delete(TaskDependency).where( + TaskDependency.workspace_id == self._ws, + TaskDependency.task_id == task_id, + ) + ) + session.flush() + for dep in _dedup(depends_on): + session.add( + TaskDependency( + workspace_id=self._ws, + task_id=task_id, + depends_on_id=dep, + created_at=_now(), + ) + ) + session.flush() + + # ------------------------------------------------------------------ # + # Epic # + # ------------------------------------------------------------------ # + + def _epic_to_dto(self, row: Epic) -> EpicDTO: + return EpicDTO( + id=row.id, + key=row.key, + project_id=row.project_id, + title=row.title, + description=row.description, + status=row.status, + spec_id=row.spec_id, + labels=list(row.labels or []), + created_at=row.created_at, + updated_at=row.updated_at, + ) + + def create_epic(self, data: EpicDTO) -> EpicDTO: + with self._sf() as session: + now = _now() + row = Epic( + id=uuid.uuid4(), + workspace_id=self._ws, + project_id=data.project_id, + key=self._next_key(session, Epic, "EPIC"), + title=data.title, + description=data.description, + status=data.status, + spec_id=data.spec_id, + labels=list(data.labels), + created_at=now, + updated_at=now, + ) + session.add(row) + session.commit() + return self._epic_to_dto(row) + + def get_epic(self, epic_id: uuid.UUID) -> EpicDTO: + with self._sf() as session: + return self._epic_to_dto(self._get(session, Epic, epic_id, "epic")) + + def update_epic(self, epic_id: uuid.UUID, data: EpicDTO) -> EpicDTO: + with self._sf() as session: + row = self._get(session, Epic, epic_id, "epic") + row.project_id = data.project_id + row.title = data.title + row.description = data.description + row.status = data.status + row.spec_id = data.spec_id + row.labels = list(data.labels) + row.updated_at = _now() + session.commit() + return self._epic_to_dto(row) + + def list_epics(self, filter: BoardFilter | None = None) -> list[EpicDTO]: + with self._sf() as session: + rows = ( + session.execute( + select(Epic) + .where(Epic.workspace_id == self._ws) + .order_by(Epic.created_at, Epic.id) + ) + .scalars() + .all() + ) + dtos = [self._epic_to_dto(r) for r in rows] + return _paginate([e for e in dtos if _epic_matches(e, filter)], filter) + + def delete_epic(self, epic_id: uuid.UUID) -> None: + with self._sf() as session: + session.delete(self._get(session, Epic, epic_id, "epic")) + session.commit() + + # ------------------------------------------------------------------ # + # Task # + # ------------------------------------------------------------------ # + + def _apply_task_fields(self, row: Task, data: TaskDTO) -> None: + row.project_id = data.project_id + row.epic_id = data.epic_id + row.spec_id = data.spec_id + row.sprint_id = data.sprint_id + row.milestone_id = data.milestone_id + row.assignee_id = data.assignee_id + row.kind = dbe.TaskKind(data.kind.value) + row.title = data.title + row.description = data.description + row.status = dbe.TaskStatus(data.status.value) + row.priority = dbe.Priority(data.priority.value) + row.estimate = data.estimate + row.execution_mode = dbe.ExecutionMode(data.execution_mode.value) + row.repo_targets = [m.model_dump(mode="json") for m in data.repo_targets] + row.instructions_profile = data.instructions_profile + row.skill_profile = data.skill_profile + row.allowed_actions = list(data.allowed_actions) + row.restricted_actions = list(data.restricted_actions) + row.requires_approval = data.requires_approval.model_dump(mode="json") + row.knowledge_scope = ( + data.knowledge_scope.model_dump(mode="json") + if data.knowledge_scope is not None + else None + ) + row.subagent_policy = data.subagent_policy.model_dump(mode="json") + row.handoff_rules = ( + data.handoff_rules.model_dump(mode="json") + if data.handoff_rules is not None + else None + ) + row.acceptance_criteria = [m.model_dump(mode="json") for m in data.acceptance_criteria] + row.labels = list(data.labels) + + def _task_to_dto(self, session: Session, row: Task) -> TaskDTO: + return TaskDTO( + id=row.id, + key=row.key, + project_id=row.project_id, + epic_id=row.epic_id, + spec_id=row.spec_id, + kind=ce.TaskKind(_ev(row.kind)), + title=row.title, + description=row.description, + status=ce.TaskStatus(_ev(row.status)), + priority=ce.Priority(_ev(row.priority)), + estimate=row.estimate, + execution_mode=ce.ExecutionMode(_ev(row.execution_mode)), + repo_targets=[RepoTarget.model_validate(x) for x in (row.repo_targets or [])], + instructions_profile=row.instructions_profile, + skill_profile=row.skill_profile, + acceptance_criteria=[ + AcceptanceCriterion.model_validate(x) for x in (row.acceptance_criteria or []) + ], + allowed_actions=list(row.allowed_actions or []), + restricted_actions=list(row.restricted_actions or []), + requires_approval=ApprovalPolicy.model_validate(row.requires_approval or {}), + knowledge_scope=( + KnowledgeScope.model_validate(row.knowledge_scope) + if row.knowledge_scope is not None + else None + ), + subagent_policy=SubAgentPolicy.model_validate(row.subagent_policy or {}), + handoff_rules=( + HandoffRules.model_validate(row.handoff_rules) + if row.handoff_rules is not None + else None + ), + labels=list(row.labels or []), + assignee_id=row.assignee_id, + sprint_id=row.sprint_id, + milestone_id=row.milestone_id, + depends_on=self._depends_on(session, row.id), + created_at=row.created_at, + updated_at=row.updated_at, + ) + + def create_task(self, data: TaskDTO) -> TaskDTO: + with self._sf() as session: + now = _now() + new_id = uuid.uuid4() + row = Task( + id=new_id, + workspace_id=self._ws, + key=self._next_key(session, Task, "TASK"), + created_at=now, + updated_at=now, + ) + self._apply_task_fields(row, data) + session.add(row) + session.flush() + self._set_edges(session, new_id, data.depends_on) + session.commit() + return self._task_to_dto(session, row) + + def get_task(self, task_id: uuid.UUID) -> TaskDTO: + with self._sf() as session: + return self._task_to_dto(session, self._get(session, Task, task_id, "task")) + + def update_task(self, task_id: uuid.UUID, data: TaskDTO) -> TaskDTO: + with self._sf() as session: + row = self._get(session, Task, task_id, "task") + edges = self._task_edges(session) + edges[task_id] = set(data.depends_on) + if has_cycle(edges): + raise CycleError( + f"updating task {task_id} dependencies would create a cycle" + ) + self._apply_task_fields(row, data) + row.updated_at = _now() + self._set_edges(session, task_id, data.depends_on) + session.commit() + return self._task_to_dto(session, row) + + def list_tasks(self, filter: BoardFilter | None = None) -> list[TaskDTO]: + with self._sf() as session: + rows = ( + session.execute( + select(Task) + .where(Task.workspace_id == self._ws) + .order_by(Task.created_at, Task.id) + ) + .scalars() + .all() + ) + dtos = [self._task_to_dto(session, r) for r in rows] + return _paginate([t for t in dtos if _task_matches(t, filter)], filter) + + def delete_task(self, task_id: uuid.UUID) -> None: + with self._sf() as session: + row = self._get(session, Task, task_id, "task") + session.execute( + delete(TaskDependency).where( + TaskDependency.workspace_id == self._ws, + (TaskDependency.task_id == task_id) + | (TaskDependency.depends_on_id == task_id), + ) + ) + session.delete(row) + session.commit() + + # ------------------------------------------------------------------ # + # Sprint # + # ------------------------------------------------------------------ # + + def _sprint_to_dto(self, row: Sprint) -> SprintDTO: + return SprintDTO( + id=row.id, + project_id=row.project_id, + name=row.name, + goal=row.goal, + starts_at=row.start_date, + ends_at=row.end_date, + task_ids=[uuid.UUID(str(t)) for t in (row.task_ids or [])], + ) + + def create_sprint(self, data: SprintDTO) -> SprintDTO: + with self._sf() as session: + now = _now() + row = Sprint( + id=uuid.uuid4(), + workspace_id=self._ws, + project_id=data.project_id, + name=data.name, + goal=data.goal, + start_date=_naive(data.starts_at), + end_date=_naive(data.ends_at), + task_ids=[str(t) for t in data.task_ids], + created_at=now, + updated_at=now, + ) + session.add(row) + session.commit() + return self._sprint_to_dto(row) + + def get_sprint(self, sprint_id: uuid.UUID) -> SprintDTO: + with self._sf() as session: + return self._sprint_to_dto(self._get(session, Sprint, sprint_id, "sprint")) + + def update_sprint(self, sprint_id: uuid.UUID, data: SprintDTO) -> SprintDTO: + with self._sf() as session: + row = self._get(session, Sprint, sprint_id, "sprint") + row.project_id = data.project_id + row.name = data.name + row.goal = data.goal + row.start_date = _naive(data.starts_at) + row.end_date = _naive(data.ends_at) + row.task_ids = [str(t) for t in data.task_ids] + row.updated_at = _now() + session.commit() + return self._sprint_to_dto(row) + + def list_sprints(self, filter: BoardFilter | None = None) -> list[SprintDTO]: + with self._sf() as session: + rows = ( + session.execute( + select(Sprint) + .where(Sprint.workspace_id == self._ws) + .order_by(Sprint.created_at, Sprint.id) + ) + .scalars() + .all() + ) + dtos = [self._sprint_to_dto(r) for r in rows] + return _paginate( + [s for s in dtos if _simple_matches(s.project_id, [s.name, s.goal], filter)], + filter, + ) + + def delete_sprint(self, sprint_id: uuid.UUID) -> None: + with self._sf() as session: + session.delete(self._get(session, Sprint, sprint_id, "sprint")) + session.commit() + + # ------------------------------------------------------------------ # + # Milestone # + # ------------------------------------------------------------------ # + + def _milestone_to_dto(self, row: Milestone) -> MilestoneDTO: + return MilestoneDTO( + id=row.id, + project_id=row.project_id, + name=row.name, + description=row.description, + due_at=row.due_date, + ) + + def create_milestone(self, data: MilestoneDTO) -> MilestoneDTO: + with self._sf() as session: + now = _now() + row = Milestone( + id=uuid.uuid4(), + workspace_id=self._ws, + project_id=data.project_id, + name=data.name, + description=data.description, + due_date=_naive(data.due_at), + created_at=now, + updated_at=now, + ) + session.add(row) + session.commit() + return self._milestone_to_dto(row) + + def get_milestone(self, milestone_id: uuid.UUID) -> MilestoneDTO: + with self._sf() as session: + return self._milestone_to_dto(self._get(session, Milestone, milestone_id, "milestone")) + + def update_milestone(self, milestone_id: uuid.UUID, data: MilestoneDTO) -> MilestoneDTO: + with self._sf() as session: + row = self._get(session, Milestone, milestone_id, "milestone") + row.project_id = data.project_id + row.name = data.name + row.description = data.description + row.due_date = _naive(data.due_at) + row.updated_at = _now() + session.commit() + return self._milestone_to_dto(row) + + def list_milestones(self, filter: BoardFilter | None = None) -> list[MilestoneDTO]: + with self._sf() as session: + rows = ( + session.execute( + select(Milestone) + .where(Milestone.workspace_id == self._ws) + .order_by(Milestone.created_at, Milestone.id) + ) + .scalars() + .all() + ) + dtos = [self._milestone_to_dto(r) for r in rows] + return _paginate( + [ + m + for m in dtos + if _simple_matches(m.project_id, [m.name, m.description], filter) + ], + filter, + ) + + def delete_milestone(self, milestone_id: uuid.UUID) -> None: + with self._sf() as session: + session.delete(self._get(session, Milestone, milestone_id, "milestone")) + session.commit() + + # ------------------------------------------------------------------ # + # Incident # + # ------------------------------------------------------------------ # + + def _incident_to_dto(self, row: Incident) -> IncidentDTO: + return IncidentDTO( + id=row.id, + key=row.key, + project_id=row.project_id, + title=row.title, + description=row.description, + severity=ce.IncidentSeverity(_ev(row.severity)), + state=ce.IncidentState(_ev(row.state)), + created_at=row.created_at, + updated_at=row.updated_at, + ) + + def create_incident(self, data: IncidentDTO) -> IncidentDTO: + with self._sf() as session: + now = _now() + row = Incident( + id=uuid.uuid4(), + workspace_id=self._ws, + project_id=data.project_id, + key=self._next_key(session, Incident, "INC"), + title=data.title, + description=data.description, + severity=dbe.IncidentSeverity(data.severity.value), + state=dbe.IncidentState(data.state.value), + created_at=now, + updated_at=now, + ) + session.add(row) + session.commit() + return self._incident_to_dto(row) + + def get_incident(self, incident_id: uuid.UUID) -> IncidentDTO: + with self._sf() as session: + return self._incident_to_dto(self._get(session, Incident, incident_id, "incident")) + + def update_incident(self, incident_id: uuid.UUID, data: IncidentDTO) -> IncidentDTO: + with self._sf() as session: + row = self._get(session, Incident, incident_id, "incident") + row.project_id = data.project_id + row.title = data.title + row.description = data.description + row.severity = dbe.IncidentSeverity(data.severity.value) + row.state = dbe.IncidentState(data.state.value) + row.updated_at = _now() + session.commit() + return self._incident_to_dto(row) + + def list_incidents(self, filter: BoardFilter | None = None) -> list[IncidentDTO]: + with self._sf() as session: + rows = ( + session.execute( + select(Incident) + .where(Incident.workspace_id == self._ws) + .order_by(Incident.created_at, Incident.id) + ) + .scalars() + .all() + ) + dtos = [self._incident_to_dto(r) for r in rows] + return _paginate( + [ + i + for i in dtos + if _simple_matches(i.project_id, [i.title, i.description], filter) + ], + filter, + ) + + def delete_incident(self, incident_id: uuid.UUID) -> None: + with self._sf() as session: + session.delete(self._get(session, Incident, incident_id, "incident")) + session.commit() + + # ------------------------------------------------------------------ # + # Cross-cutting: status, bulk, dependencies # + # ------------------------------------------------------------------ # + + def set_status(self, task_id: uuid.UUID, status: TaskStatus) -> TaskDTO: + with self._sf() as session: + row = self._get(session, Task, task_id, "task") + validate_transition(ce.TaskStatus(_ev(row.status)), status) + row.status = dbe.TaskStatus(status.value) + row.updated_at = _now() + session.commit() + return self._task_to_dto(session, row) + + def bulk_update(self, updates: list[BulkUpdate]) -> list[TaskDTO]: + with self._sf() as session: + # Two-pass for atomicity: validate everything before mutating anything. + planned: list[tuple[BulkUpdate, Task]] = [] + for update in updates: + row = self._get(session, Task, update.task_id, "task") + if update.status is not None: + validate_transition(ce.TaskStatus(_ev(row.status)), update.status) + planned.append((update, row)) + + now = _now() + for update, row in planned: + if update.status is not None: + row.status = dbe.TaskStatus(update.status.value) + if update.priority is not None: + row.priority = dbe.Priority(update.priority.value) + if update.assignee_id is not None: + row.assignee_id = update.assignee_id + if update.sprint_id is not None: + row.sprint_id = update.sprint_id + if update.labels is not None: + row.labels = list(update.labels) + row.updated_at = now + session.flush() + results = [self._task_to_dto(session, row) for _u, row in planned] + session.commit() + return results + + def dependency_add(self, task_id: uuid.UUID, depends_on_id: uuid.UUID) -> None: + with self._sf() as session: + self._get(session, Task, task_id, "task") + self._get(session, Task, depends_on_id, "task") + if would_create_cycle(self._task_edges(session), task_id, depends_on_id): + raise CycleError( + f"task {task_id} depending on {depends_on_id} would create a cycle" + ) + existing = session.execute( + select(TaskDependency).where( + TaskDependency.workspace_id == self._ws, + TaskDependency.task_id == task_id, + TaskDependency.depends_on_id == depends_on_id, + ) + ).scalar_one_or_none() + if existing is None: + session.add( + TaskDependency( + workspace_id=self._ws, + task_id=task_id, + depends_on_id=depends_on_id, + created_at=_now(), + ) + ) + task = session.get(Task, task_id) + task.updated_at = _now() + session.commit() + + +__all__ = ["SqlAlchemyBoardService"] diff --git a/packages/board-core/tests/test_sql_board_service.py b/packages/board-core/tests/test_sql_board_service.py new file mode 100644 index 00000000..75295256 --- /dev/null +++ b/packages/board-core/tests/test_sql_board_service.py @@ -0,0 +1,602 @@ +"""Postgres integration tests for :class:`SqlAlchemyBoardService` (F01 persistence). + +Exercises the DB-backed board repository against a real pgvector Postgres via the +shared ``pg_engine`` fixture (root ``conftest.py``): a full DTO round-trip for +every entity, the status workflow, bulk atomicity, the dependency graph + cycle +detection, saved-filter queries (filtering + ordering + pagination), key + +workspace uniqueness constraints, cross-workspace isolation, durability across +service instances, and structural conformance to the frozen ``BoardService`` +protocol. Skips cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. + +Each behaviour mirrors ``tests/test_board_service.py`` (the in-memory contract), +so both backends are proven to satisfy the same protocol identically. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_board import SqlAlchemyBoardService +from forge_board.exceptions import EntityNotFoundError, InvalidStatusTransitionError +from forge_contracts import ( + BoardFilter, + BoardService, + BulkUpdate, + CycleError, + EpicDTO, + IncidentDTO, + KnowledgeScope, + MilestoneDTO, + Priority, + RepoTarget, + SprintDTO, + TaskDTO, + TaskKind, + TaskStatus, +) +from forge_contracts.enums import IncidentSeverity, IncidentState +from forge_db.base import Base +from forge_db.models import Project, Task, TaskDependency, Workspace + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def seed(factory: sessionmaker[Session]) -> dict[str, uuid.UUID]: + """A workspace + two projects (A/B) + a second isolated workspace.""" + ws = uuid.uuid4() + other_ws = uuid.uuid4() + proj_a = uuid.uuid4() + proj_b = uuid.uuid4() + other_proj = uuid.uuid4() + with factory() as session: + session.add(Workspace(id=ws, name="Acme", slug=f"acme-{uuid.uuid4().hex[:8]}")) + session.add(Workspace(id=other_ws, name="Other", slug=f"other-{uuid.uuid4().hex[:8]}")) + session.flush() + session.add(Project(id=proj_a, workspace_id=ws, name="A", key=f"A{uuid.uuid4().hex[:4]}")) + session.add(Project(id=proj_b, workspace_id=ws, name="B", key=f"B{uuid.uuid4().hex[:4]}")) + session.add( + Project(id=other_proj, workspace_id=other_ws, name="O", key=f"O{uuid.uuid4().hex[:4]}") + ) + session.commit() + return { + "ws": ws, + "other_ws": other_ws, + "proj_a": proj_a, + "proj_b": proj_b, + "other_proj": other_proj, + } + + +@pytest.fixture +def svc( + factory: sessionmaker[Session], seed: dict[str, uuid.UUID] +) -> SqlAlchemyBoardService: + return SqlAlchemyBoardService(factory, seed["ws"]) + + +def _task( + svc: SqlAlchemyBoardService, project: uuid.UUID, title: str = "Do thing", **kw +) -> TaskDTO: + return svc.create_task(TaskDTO(title=title, project_id=project, **kw)) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_service_conforms_to_board_service_protocol(svc: SqlAlchemyBoardService) -> None: + assert isinstance(svc, BoardService) + + +# --------------------------------------------------------------------------- # +# Task CRUD + full round-trip # +# --------------------------------------------------------------------------- # + + +def test_create_task_assigns_id_key_and_timestamps( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + task = _task(svc, seed["proj_a"], "Implement login") + assert task.id is not None + assert task.key == "TASK-1" + assert task.created_at is not None + assert task.updated_at is not None + assert task.status is TaskStatus.BACKLOG + + +def test_keys_increment_per_creation( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + t1 = _task(svc, seed["proj_a"], "one") + t2 = _task(svc, seed["proj_a"], "two") + assert t1.key == "TASK-1" + assert t2.key == "TASK-2" + + +def test_task_full_round_trip( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + """Every DTO field — including nested models + None-able scopes — survives.""" + created = _task( + svc, + seed["proj_a"], + "round trip", + description="body", + kind=TaskKind.BUG, + priority=Priority.HIGH, + estimate=5, + labels=["x", "y"], + repo_targets=[RepoTarget(repo="svc-a", base_branch="develop")], + knowledge_scope=KnowledgeScope(repos=["r1"], freshness_min_hours=12), + handoff_rules=None, + ) + assert created.id is not None + fetched = svc.get_task(created.id) + assert fetched.id == created.id + assert fetched.title == "round trip" + assert fetched.description == "body" + assert fetched.kind is TaskKind.BUG + assert fetched.priority is Priority.HIGH + assert fetched.estimate == 5 + assert fetched.labels == ["x", "y"] + assert fetched.repo_targets[0].repo == "svc-a" + assert fetched.repo_targets[0].base_branch == "develop" + assert fetched.knowledge_scope is not None + assert fetched.knowledge_scope.repos == ["r1"] + assert fetched.knowledge_scope.freshness_min_hours == 12 + # ``handoff_rules=None`` round-trips as None (distinct from an empty object). + assert fetched.handoff_rules is None + + +def test_get_missing_task_raises(svc: SqlAlchemyBoardService) -> None: + with pytest.raises(EntityNotFoundError): + svc.get_task(uuid.uuid4()) + + +def test_update_task_preserves_identity_and_bumps_updated_at( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + created = _task(svc, seed["proj_a"], "original") + assert created.id is not None + edit = created.model_copy(deep=True) + edit.title = "edited" + edit.priority = Priority.HIGH + updated = svc.update_task(created.id, edit) + assert updated.id == created.id + assert updated.key == created.key # key is immutable across updates + assert updated.created_at == created.created_at + assert updated.title == "edited" + assert updated.priority is Priority.HIGH + + +def test_update_missing_task_raises(svc: SqlAlchemyBoardService) -> None: + with pytest.raises(EntityNotFoundError): + svc.update_task(uuid.uuid4(), TaskDTO(title="ghost")) + + +def test_delete_task(svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID]) -> None: + created = _task(svc, seed["proj_a"], "to delete") + assert created.id is not None + svc.delete_task(created.id) + with pytest.raises(EntityNotFoundError): + svc.get_task(created.id) + + +def test_persistence_across_service_instances( + factory: sessionmaker[Session], seed: dict[str, uuid.UUID] +) -> None: + """State is real Postgres — a fresh service instance sees prior writes.""" + svc1 = SqlAlchemyBoardService(factory, seed["ws"]) + created = _task(svc1, seed["proj_a"], "durable") + assert created.id is not None + svc2 = SqlAlchemyBoardService(factory, seed["ws"]) + assert svc2.get_task(created.id).title == "durable" + + +# --------------------------------------------------------------------------- # +# Status workflow # +# --------------------------------------------------------------------------- # + + +def test_set_status_valid_transition( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + task = _task(svc, seed["proj_a"], "status") + assert task.id is not None + moved = svc.set_status(task.id, TaskStatus.READY) + assert moved.status is TaskStatus.READY + assert moved.updated_at is not None + + +def test_set_status_invalid_transition_raises( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + task = _task(svc, seed["proj_a"], "status") + assert task.id is not None + with pytest.raises(InvalidStatusTransitionError): + svc.set_status(task.id, TaskStatus.DONE) + + +def test_set_status_missing_task_raises(svc: SqlAlchemyBoardService) -> None: + with pytest.raises(EntityNotFoundError): + svc.set_status(uuid.uuid4(), TaskStatus.READY) + + +# --------------------------------------------------------------------------- # +# Bulk update # +# --------------------------------------------------------------------------- # + + +def test_bulk_update_applies_status_and_priority( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + t1 = _task(svc, seed["proj_a"], "b1") + t2 = _task(svc, seed["proj_a"], "b2") + assert t1.id is not None and t2.id is not None + result = svc.bulk_update( + [ + BulkUpdate(task_id=t1.id, status=TaskStatus.READY, priority=Priority.URGENT), + BulkUpdate(task_id=t2.id, status=TaskStatus.READY), + ] + ) + assert {r.status for r in result} == {TaskStatus.READY} + assert svc.get_task(t1.id).priority is Priority.URGENT + + +def test_bulk_update_is_atomic_on_invalid_transition( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + t1 = _task(svc, seed["proj_a"], "ok") + t2 = _task(svc, seed["proj_a"], "bad") + assert t1.id is not None and t2.id is not None + with pytest.raises(InvalidStatusTransitionError): + svc.bulk_update( + [ + BulkUpdate(task_id=t1.id, status=TaskStatus.READY), + BulkUpdate(task_id=t2.id, status=TaskStatus.DONE), # illegal jump + ] + ) + # Nothing applied: t1 still in its original status (rolled back). + assert svc.get_task(t1.id).status is TaskStatus.BACKLOG + + +def test_bulk_update_missing_task_is_atomic( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + t1 = _task(svc, seed["proj_a"], "ok") + assert t1.id is not None + with pytest.raises(EntityNotFoundError): + svc.bulk_update( + [ + BulkUpdate(task_id=t1.id, status=TaskStatus.READY), + BulkUpdate(task_id=uuid.uuid4(), status=TaskStatus.READY), + ] + ) + assert svc.get_task(t1.id).status is TaskStatus.BACKLOG + + +# --------------------------------------------------------------------------- # +# Dependencies + cycle detection # +# --------------------------------------------------------------------------- # + + +def test_dependency_add_records_edge( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + b = _task(svc, seed["proj_a"], "b") + assert a.id is not None and b.id is not None + svc.dependency_add(a.id, b.id) + assert b.id in svc.get_task(a.id).depends_on + + +def test_dependency_add_is_idempotent( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + b = _task(svc, seed["proj_a"], "b") + assert a.id is not None and b.id is not None + svc.dependency_add(a.id, b.id) + svc.dependency_add(a.id, b.id) # no duplicate, no error + assert svc.get_task(a.id).depends_on == [b.id] + + +def test_dependency_cycle_raises( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + b = _task(svc, seed["proj_a"], "b") + assert a.id is not None and b.id is not None + svc.dependency_add(a.id, b.id) + with pytest.raises(CycleError): + svc.dependency_add(b.id, a.id) + + +def test_transitive_dependency_cycle_raises( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + b = _task(svc, seed["proj_a"], "b") + c = _task(svc, seed["proj_a"], "c") + assert a.id is not None and b.id is not None and c.id is not None + svc.dependency_add(a.id, b.id) + svc.dependency_add(b.id, c.id) + with pytest.raises(CycleError): + svc.dependency_add(c.id, a.id) + + +def test_self_dependency_raises( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + assert a.id is not None + with pytest.raises(CycleError): + svc.dependency_add(a.id, a.id) + + +def test_dependency_add_missing_task_raises( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + assert a.id is not None + with pytest.raises(EntityNotFoundError): + svc.dependency_add(a.id, uuid.uuid4()) + + +def test_update_task_with_dependency_cycle_raises( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + a = _task(svc, seed["proj_a"], "a") + b = _task(svc, seed["proj_a"], "b") + assert a.id is not None and b.id is not None + svc.dependency_add(a.id, b.id) + edit = svc.get_task(b.id) + edit.depends_on = [a.id] + with pytest.raises(CycleError): + svc.update_task(b.id, edit) + # The rejected edge was not persisted. + assert svc.get_task(b.id).depends_on == [] + + +def test_delete_task_cascades_dependency_edges( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + a = _task(svc, seed["proj_a"], "a") + b = _task(svc, seed["proj_a"], "b") + assert a.id is not None and b.id is not None + svc.dependency_add(a.id, b.id) + svc.delete_task(a.id) + with factory() as session: + assert session.query(TaskDependency).count() == 0 + + +# --------------------------------------------------------------------------- # +# Filters / saved-filter queries (filtering + ordering + pagination) # +# --------------------------------------------------------------------------- # + + +def test_list_tasks_filter_by_status( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + t1 = _task(svc, seed["proj_a"], "one") + _task(svc, seed["proj_a"], "two") + assert t1.id is not None + svc.set_status(t1.id, TaskStatus.READY) + ready = svc.list_tasks(BoardFilter(statuses=[TaskStatus.READY.value])) + assert [t.id for t in ready] == [t1.id] + + +def test_list_tasks_filter_by_project( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + _task(svc, seed["proj_a"], "a") + _task(svc, seed["proj_b"], "b") + only_b = svc.list_tasks(BoardFilter(project_id=seed["proj_b"])) + assert len(only_b) == 1 + assert only_b[0].project_id == seed["proj_b"] + + +def test_list_tasks_filter_by_text( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + _task(svc, seed["proj_a"], "Implement OAuth login") + _task(svc, seed["proj_a"], "Fix flaky test") + hits = svc.list_tasks(BoardFilter(text="oauth")) + assert len(hits) == 1 + assert "OAuth" in hits[0].title + + +def test_list_tasks_filter_by_kind_priority_and_labels( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + _task(svc, seed["proj_a"], "bug", kind=TaskKind.BUG, priority=Priority.HIGH, labels=["red"]) + _task(svc, seed["proj_a"], "feature", kind=TaskKind.FEATURE, priority=Priority.LOW) + assert len(svc.list_tasks(BoardFilter(kinds=[TaskKind.BUG]))) == 1 + assert len(svc.list_tasks(BoardFilter(priorities=[Priority.HIGH]))) == 1 + assert len(svc.list_tasks(BoardFilter(labels=["red"]))) == 1 + + +def test_list_tasks_ordering_is_creation_order( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + created = [_task(svc, seed["proj_a"], f"t{i}") for i in range(5)] + listed = svc.list_tasks() + assert [t.id for t in listed] == [t.id for t in created] + + +def test_list_tasks_pagination( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + created = [_task(svc, seed["proj_a"], f"t{i}") for i in range(5)] + page = svc.list_tasks(BoardFilter(limit=2, offset=1)) + assert [t.id for t in page] == [created[1].id, created[2].id] + + +def test_list_tasks_no_filter_returns_all( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + _task(svc, seed["proj_a"], "a") + _task(svc, seed["proj_a"], "b") + assert len(svc.list_tasks()) == 2 + + +# --------------------------------------------------------------------------- # +# Epic / Sprint / Milestone / Incident CRUD + round-trip # +# --------------------------------------------------------------------------- # + + +def test_epic_crud(svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID]) -> None: + spec_id = uuid.uuid4() + epic = svc.create_epic( + EpicDTO(title="Auth epic", project_id=seed["proj_a"], labels=["auth"], spec_id=spec_id) + ) + assert epic.id is not None + assert epic.key == "EPIC-1" + fetched = svc.get_epic(epic.id) + assert fetched.title == "Auth epic" + assert fetched.labels == ["auth"] + assert fetched.spec_id == spec_id + edit = fetched.model_copy(deep=True) + edit.title = "Auth epic v2" + updated = svc.update_epic(epic.id, edit) + assert updated.title == "Auth epic v2" + assert updated.key == "EPIC-1" + assert len(svc.list_epics()) == 1 + svc.delete_epic(epic.id) + with pytest.raises(EntityNotFoundError): + svc.get_epic(epic.id) + + +def test_sprint_crud_round_trips_task_ids( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + t1 = _task(svc, seed["proj_a"], "t1") + t2 = _task(svc, seed["proj_a"], "t2") + assert t1.id is not None and t2.id is not None + sprint = svc.create_sprint( + SprintDTO( + name="Sprint 1", + project_id=seed["proj_a"], + task_ids=[t1.id, t2.id], + starts_at=datetime(2026, 7, 1, tzinfo=UTC), + ) + ) + assert sprint.id is not None + fetched = svc.get_sprint(sprint.id) + assert fetched.name == "Sprint 1" + assert fetched.task_ids == [t1.id, t2.id] + edit = fetched.model_copy(deep=True) + edit.goal = "ship board" + assert svc.update_sprint(sprint.id, edit).goal == "ship board" + assert len(svc.list_sprints()) == 1 + svc.delete_sprint(sprint.id) + with pytest.raises(EntityNotFoundError): + svc.get_sprint(sprint.id) + + +def test_milestone_crud(svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID]) -> None: + ms = svc.create_milestone(MilestoneDTO(name="GA", project_id=seed["proj_a"])) + assert ms.id is not None + assert svc.get_milestone(ms.id).name == "GA" + edit = ms.model_copy(deep=True) + edit.description = "general availability" + assert svc.update_milestone(ms.id, edit).description == "general availability" + assert len(svc.list_milestones()) == 1 + svc.delete_milestone(ms.id) + with pytest.raises(EntityNotFoundError): + svc.get_milestone(ms.id) + + +def test_incident_crud_and_key( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + inc = svc.create_incident( + IncidentDTO( + title="DB down", + project_id=seed["proj_a"], + severity=IncidentSeverity.HIGH, + state=IncidentState.ALERT_RECEIVED, + ) + ) + assert inc.id is not None + assert inc.key == "INC-1" + fetched = svc.get_incident(inc.id) + assert fetched.title == "DB down" + assert fetched.severity is IncidentSeverity.HIGH + edit = fetched.model_copy(deep=True) + edit.title = "DB degraded" + edit.state = IncidentState.CONTEXT_GATHERING + updated = svc.update_incident(inc.id, edit) + assert updated.title == "DB degraded" + assert updated.state is IncidentState.CONTEXT_GATHERING + # A second incident with an unset dedup_key is allowed (partial index exempts NULL). + inc2 = svc.create_incident(IncidentDTO(title="Second", project_id=seed["proj_a"])) + assert inc2.key == "INC-2" + assert len(svc.list_incidents()) == 2 + svc.delete_incident(inc.id) + with pytest.raises(EntityNotFoundError): + svc.get_incident(inc.id) + + +def test_list_epics_filter_by_project( + svc: SqlAlchemyBoardService, seed: dict[str, uuid.UUID] +) -> None: + svc.create_epic(EpicDTO(title="a", project_id=seed["proj_a"])) + svc.create_epic(EpicDTO(title="b", project_id=seed["proj_b"])) + assert len(svc.list_epics(BoardFilter(project_id=seed["proj_a"]))) == 1 + + +# --------------------------------------------------------------------------- # +# Constraints + cross-workspace isolation # +# --------------------------------------------------------------------------- # + + +def test_cross_workspace_isolation( + factory: sessionmaker[Session], seed: dict[str, uuid.UUID] +) -> None: + svc_a = SqlAlchemyBoardService(factory, seed["ws"]) + svc_other = SqlAlchemyBoardService(factory, seed["other_ws"]) + mine = _task(svc_a, seed["proj_a"], "mine") + assert mine.id is not None + # The other workspace cannot see, fetch, or mutate it (a foreign id is 404). + with pytest.raises(EntityNotFoundError): + svc_other.get_task(mine.id) + assert svc_other.list_tasks() == [] + # Keys are per-workspace: the other workspace starts its own TASK-1 sequence. + theirs = _task(svc_other, seed["other_proj"], "theirs") + assert theirs.key == "TASK-1" + + +def test_key_unique_per_workspace( + factory: sessionmaker[Session], seed: dict[str, uuid.UUID] +) -> None: + """The (workspace_id, key) unique constraint is enforced by Postgres.""" + with factory() as session: + session.add( + Task(workspace_id=seed["ws"], project_id=seed["proj_a"], key="TASK-1", title="x") + ) + session.commit() + session.add( + Task(workspace_id=seed["ws"], project_id=seed["proj_a"], key="TASK-1", title="y") + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() diff --git a/packages/db/forge_db/models/__init__.py b/packages/db/forge_db/models/__init__.py index c50cd285..c5da6b51 100644 --- a/packages/db/forge_db/models/__init__.py +++ b/packages/db/forge_db/models/__init__.py @@ -113,6 +113,7 @@ SpecDocument, Sprint, Task, + TaskDependency, ) from forge_db.models.platform_api_key import PlatformAPIKey from forge_db.models.pm import PMConnection, PMTaskLink, PMWebhookDelivery @@ -277,6 +278,7 @@ "SubAgentRun", "SyncMode", "Task", + "TaskDependency", "TaskKind", "TaskStatus", "Team", diff --git a/packages/db/forge_db/models/planning.py b/packages/db/forge_db/models/planning.py index 8797bfa5..939872bb 100644 --- a/packages/db/forge_db/models/planning.py +++ b/packages/db/forge_db/models/planning.py @@ -53,6 +53,12 @@ class Epic(WorkspaceScopedModel): priority: Mapped[Priority] = mapped_column( enum_type(Priority), default=Priority.MEDIUM, nullable=False ) + # F01 board persistence: carries the ``EpicDTO`` free-form fields the domain + # service round-trips but the base Epic has no dedicated column for. ``spec_id`` + # is a plain UUID mirror of the DTO value (the referential 1:1 link lives on + # ``spec_document.epic_id``); ``labels`` is the DTO's saved-filter label set. + spec_id: Mapped[uuid.UUID | None] = mapped_column(Uuid(as_uuid=True), nullable=True) + labels: Mapped[list[Any]] = mapped_column(json_type(), default=list, nullable=False) project: Mapped[Project] = relationship(back_populates="epics") spec_document: Mapped[SpecDocument | None] = relationship( @@ -143,6 +149,10 @@ class Sprint(WorkspaceScopedModel): committed_task_ids: Mapped[list[Any]] = mapped_column(json_type(), default=list, nullable=False) position: Mapped[str | None] = mapped_column(Text, nullable=True) velocity_version: Mapped[int] = mapped_column(BigInteger, default=0, nullable=False) + # F01 board persistence: the ``SprintDTO.task_ids`` membership list the domain + # service round-trips verbatim (distinct from the F26 ``committed_task_ids`` + # velocity snapshot frozen at sprint start). + task_ids: Mapped[list[Any]] = mapped_column(json_type(), default=list, nullable=False) project: Mapped[Project] = relationship(back_populates="sprints") tasks: Mapped[list[Task]] = relationship(back_populates="sprint") @@ -227,6 +237,9 @@ class Task(WorkspaceScopedModel): acceptance_criteria: Mapped[list[Any]] = mapped_column( json_type(), default=list, nullable=False ) + # F01 board persistence: the ``TaskDTO.labels`` saved-filter set (the DTO's + # ``depends_on`` edges live in the ``task_dependency`` adjacency table below). + labels: Mapped[list[Any]] = mapped_column(json_type(), default=list, nullable=False) project: Mapped[Project] = relationship(back_populates="tasks") epic: Mapped[Epic | None] = relationship(back_populates="tasks") @@ -282,3 +295,32 @@ class Incident(WorkspaceScopedModel): resolved_at: Mapped[datetime | None] = mapped_column(nullable=True) project: Mapped[Project] = relationship(back_populates="incidents") + + +class TaskDependency(WorkspaceScopedModel): + """A directed task-dependency edge (``TaskDTO.depends_on``). + + An edge ``(task_id -> depends_on_id)`` means *task ``task_id`` depends on / + is blocked-by task ``depends_on_id``* — the same orientation the in-memory + board's cycle detector uses. This adjacency table is what the DB-backed + :class:`~forge_board.sql_service.SqlAlchemyBoardService` persists the + dependency graph in (the base ``task`` table has no ``depends_on`` column). + Both endpoints ``ON DELETE CASCADE`` so deleting a task removes its incident + edges without leaving dangling references. + """ + + __tablename__ = "task_dependency" + __table_args__ = (UniqueConstraint("task_id", "depends_on_id"),) + + task_id: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("task.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + depends_on_id: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("task.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) diff --git a/packages/db/migrations/versions/0001_baseline.py b/packages/db/migrations/versions/0001_baseline.py index 81621056..a1739e67 100644 --- a/packages/db/migrations/versions/0001_baseline.py +++ b/packages/db/migrations/versions/0001_baseline.py @@ -75,6 +75,9 @@ # F35 benchmark-leaderboard — created by 0018_f35_benchmark_leaderboard. "benchmark_suite", "benchmark_submission", + # F01 board persistence (task dependency adjacency) — created by + # 0024_board_persistence. + "task_dependency", } ) diff --git a/packages/db/migrations/versions/0024_board_persistence.py b/packages/db/migrations/versions/0024_board_persistence.py new file mode 100644 index 00000000..36cd498c --- /dev/null +++ b/packages/db/migrations/versions/0024_board_persistence.py @@ -0,0 +1,79 @@ +"""f01 board persistence: DTO round-trip columns + task-dependency adjacency + +Backs the DB-backed :class:`forge_board.sql_service.SqlAlchemyBoardService` with +the storage the in-memory board's DTOs need but the base planning tables lacked: + +* additive ``epic.spec_id`` + ``epic.labels`` columns (the ``EpicDTO`` fields + with no dedicated column); +* additive ``task.labels`` column (the ``TaskDTO`` saved-filter label set); +* additive ``sprint.task_ids`` column (the ``SprintDTO`` membership list, distinct + from the F26 ``committed_task_ids`` velocity snapshot); +* the ``task_dependency`` adjacency table (``TaskDTO.depends_on`` edges), deferred + from the baseline and created/dropped here from metadata (so the cross-dialect + column variants apply automatically). + +Foundation note (mirrors 0007/0008): ``forge_db``'s baseline is metadata-driven, +so a fresh chain already provisions the new columns from the model. To stay +idiomatic *and* own an explicit, reversible step this migration is **idempotent**: +``upgrade`` adds only what is missing, ``downgrade`` drops only what F01-persist +introduced. + +Revision ID: 0024_board_persistence +Revises: 0023_envelope_key_version +Create Date: 2026-07-05 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) +from forge_db.base import Base + +# revision identifiers, used by Alembic. +revision: str = "0024_board_persistence" +down_revision: str | None = "0023_envelope_key_version" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_DEP_TABLE = "task_dependency" + +# JSON column type matching ``forge_db.base.json_type`` (JSONB on Postgres). +_JSON = sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql") + +# (table, column-name, column) additive columns owned by this revision, ordered +# so the downgrade drops them in reverse. +_COLUMNS: tuple[tuple[str, str, sa.Column], ...] = ( + ("epic", "spec_id", sa.Column("spec_id", sa.Uuid(), nullable=True)), + ("epic", "labels", sa.Column("labels", _JSON, server_default="[]", nullable=False)), + ("task", "labels", sa.Column("labels", _JSON, server_default="[]", nullable=False)), + ("sprint", "task_ids", sa.Column("task_ids", _JSON, server_default="[]", nullable=False)), +) + + +def _dep_table() -> list: + by_name = {t.name: t for t in Base.metadata.sorted_tables} + return [by_name[_DEP_TABLE]] if _DEP_TABLE in by_name else [] + + +def _existing_columns(table: str) -> set[str]: + return {c["name"] for c in sa.inspect(op.get_bind()).get_columns(table)} + + +def upgrade() -> None: + for table, name, column in _COLUMNS: + if name not in _existing_columns(table): + op.add_column(table, column) + + Base.metadata.create_all(bind=op.get_bind(), tables=_dep_table()) + + +def downgrade() -> None: + Base.metadata.drop_all(bind=op.get_bind(), tables=_dep_table()) + + for table, name, _column in reversed(_COLUMNS): + if name in _existing_columns(table): + op.drop_column(table, name) diff --git a/packages/db/tests/test_models.py b/packages/db/tests/test_models.py index fa5cf99d..a842c982 100644 --- a/packages/db/tests/test_models.py +++ b/packages/db/tests/test_models.py @@ -132,6 +132,9 @@ "ModelPrice", # F39 audit-log: per-workspace hash-chain cursor (audit_log itself is F30's). "AuditChainHead", + # F01 board persistence: task-dependency adjacency (depends_on edges) backing + # the DB-backed SqlAlchemyBoardService. + "TaskDependency", ] # Tables that are NOT the tenant root and therefore must carry a workspace FK. From ffa47b69b9bc0eafb8811b5e5a664f58f4a97004 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 11:13:27 +0200 Subject: [PATCH 02/12] feat(db/audit-store): Postgres persistence Co-Authored-By: Claude Fable 5 --- apps/api/forge_api/observability/__init__.py | 8 + apps/api/forge_api/observability/audit.py | 5 +- apps/api/forge_api/observability/audit_db.py | 266 ++++++++++++++++ apps/api/forge_api/observability/service.py | 6 +- apps/api/forge_api/routers/mcp.py | 7 +- apps/api/forge_api/settings.py | 7 + apps/api/tests/test_audit_store_db.py | 288 ++++++++++++++++++ packages/db/forge_db/models/__init__.py | 6 + .../db/forge_db/models/observability_audit.py | 127 ++++++++ .../0025_observability_audit_store.py | 73 +++++ packages/db/tests/test_models.py | 10 + 11 files changed, 799 insertions(+), 4 deletions(-) create mode 100644 apps/api/forge_api/observability/audit_db.py create mode 100644 apps/api/tests/test_audit_store_db.py create mode 100644 packages/db/forge_db/models/observability_audit.py create mode 100644 packages/db/migrations/versions/0025_observability_audit_store.py diff --git a/apps/api/forge_api/observability/__init__.py b/apps/api/forge_api/observability/__init__.py index 085d91e7..be8abad8 100644 --- a/apps/api/forge_api/observability/__init__.py +++ b/apps/api/forge_api/observability/__init__.py @@ -21,6 +21,11 @@ compute_payload_hash, verify_chain, ) +from forge_api.observability.audit_db import ( + DbAuditStore, + build_audit_store, + default_audit_log, +) from forge_api.observability.otel import ( OTEL_AVAILABLE, SpanRecord, @@ -51,6 +56,7 @@ "AuditEntry", "AuditLog", "AuditStore", + "DbAuditStore", "InMemoryAuditStore", "MCPAuditSink", "ObservabilityService", @@ -59,7 +65,9 @@ "RunTraceAssembler", "SpanRecord", "SpanRecorder", + "build_audit_store", "compute_payload_hash", + "default_audit_log", "get_observability_service", "get_span_recorder", "get_tracer", diff --git a/apps/api/forge_api/observability/audit.py b/apps/api/forge_api/observability/audit.py index 7a8ba9dc..05dcf928 100644 --- a/apps/api/forge_api/observability/audit.py +++ b/apps/api/forge_api/observability/audit.py @@ -187,7 +187,10 @@ class AuditLog: """Writer facade over an :class:`AuditStore` with redaction + hashing.""" def __init__(self, store: AuditStore | None = None) -> None: - self._store: AuditStore = store or InMemoryAuditStore() + # Select the passed store by identity (``is not None``), not truthiness: + # a durable store that is merely *empty* must never be mistaken for "no + # store" and silently replaced with an in-memory one. + self._store: AuditStore = store if store is not None else InMemoryAuditStore() @property def store(self) -> AuditStore: diff --git a/apps/api/forge_api/observability/audit_db.py b/apps/api/forge_api/observability/audit_db.py new file mode 100644 index 00000000..7be71002 --- /dev/null +++ b/apps/api/forge_api/observability/audit_db.py @@ -0,0 +1,266 @@ +"""Postgres-backed observability audit store (audit-store persistence). + +:class:`DbAuditStore` is a drop-in, durable alternative to +:class:`~forge_api.observability.audit.InMemoryAuditStore` that satisfies the +**same** :class:`~forge_api.observability.audit.AuditStore` protocol +(``append`` / ``all`` / ``query`` / ``verify_integrity``) — so the composition +root swaps it in behind ``FORGE_AUDIT_BACKEND=db`` with no behavioural change. +The default stays ``memory`` and the in-memory store remains the unit-test +default; this is the sink the MCP db-path +(``FORGE_MCP_AUDIT_BACKEND=db`` → ``MCPAuditSink(AuditLog())``) forwards to, so +with both flags set every live MCP call finally lands in durable Postgres. + +Behaviour parity is exact and *intentional*: + +* the global, 0-based, monotonic ``seq`` (spanning every workspace) is preserved + by a single ``observability_audit_chain_head`` cursor row locked ``FOR UPDATE`` + on append — the DB analogue of the in-memory ``len(entries)`` scheme; +* the tamper-evident hash chain re-uses the store's own + :func:`~forge_api.observability.audit._hash_entry` / + :func:`~forge_api.observability.audit.verify_chain`, so a round-tripped row + re-hashes identically and ``verify_integrity`` walks the same chain; +* ``query`` reproduces the in-memory filter/limit semantics byte-for-byte + (including the "``limit`` is the *most recent* N, ``limit==0`` → empty, + negative ``limit`` → ignored" edge cases). + +The one fidelity detail is the entry ``timestamp``: it is normalised to +timezone-aware UTC on the way *in* (before hashing + persisting) and again on the +way *out*, so the ``timestamptz`` round-trip reproduces the exact datetime that +was hashed on every dialect — the chain never breaks from an offset/representation +drift. Because the chain lives in Postgres, independently constructed +``DbAuditStore`` instances (e.g. the observability service and the MCP sink) all +converge on one durable, shared trail. +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import func, select +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.observability.audit import ( + GENESIS_HASH, + AuditCategory, + AuditEntry, + AuditLog, + AuditStore, + InMemoryAuditStore, + _hash_entry, + verify_chain, +) +from forge_db.models.observability_audit import ( + CHAIN_HEAD_ID, + ObservabilityAuditChainHead, + ObservabilityAuditEntry, +) + +__all__ = ["DbAuditStore", "build_audit_store", "default_audit_log"] + + +def _to_utc(value: datetime) -> datetime: + """Normalise a datetime to timezone-aware UTC (stable across store + DB). + + Applied identically before hashing/persisting and after reading back, so the + ``timestamptz`` round-trip reproduces the exact instant that was hashed — a + naive value is assumed UTC; an aware value is converted. + """ + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +class DbAuditStore: + """A Postgres-backed, append-only audit store (implements ``AuditStore``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + # ------------------------------------------------------------------ # + # Mapping # + # ------------------------------------------------------------------ # + + def _to_entry(self, row: ObservabilityAuditEntry) -> AuditEntry: + """Rebuild the exact :class:`AuditEntry` that produced ``row`` (re-hashable).""" + return AuditEntry( + seq=row.seq, + id=row.entry_id, + timestamp=_to_utc(row.occurred_at), + category=AuditCategory(row.category), + action=row.action, + actor=row.actor, + workspace_id=row.workspace_ref, + run_id=row.run_id, + target=row.target, + connection_id=row.connection_id, + status=row.status, + detail=row.detail, + payload_hash=row.payload_hash, + latency_ms=row.latency_ms, + metadata=dict(row.entry_metadata or {}), + redacted=row.redacted, + prev_hash=row.prev_hash, + entry_hash=row.entry_hash, + ) + + # ------------------------------------------------------------------ # + # AuditStore protocol # + # ------------------------------------------------------------------ # + + def append(self, entry: AuditEntry) -> AuditEntry: + """Stamp ``entry`` into the global chain and persist it durably.""" + # Canonicalise the timestamp up front so the hash we compute here matches + # the one recomputed after a timestamptz round-trip. + canonical = entry.model_copy(update={"timestamp": _to_utc(entry.timestamp)}) + with self._sf() as session: + head = session.execute( + select(ObservabilityAuditChainHead) + .where(ObservabilityAuditChainHead.id == CHAIN_HEAD_ID) + .with_for_update() + ).scalar_one_or_none() + if head is None: + head = ObservabilityAuditChainHead( + id=CHAIN_HEAD_ID, last_seq=-1, last_hash=GENESIS_HASH + ) + session.add(head) + session.flush() + + seq = head.last_seq + 1 + prev = head.last_hash + stamped = canonical.model_copy( + update={"seq": seq, "prev_hash": prev, "entry_hash": None} + ) + entry_hash = _hash_entry(stamped) + final = stamped.model_copy(update={"entry_hash": entry_hash}) + + session.add( + ObservabilityAuditEntry( + entry_id=final.id, + seq=seq, + occurred_at=final.timestamp, + category=final.category.value, + action=final.action, + actor=final.actor, + workspace_ref=final.workspace_id, + run_id=final.run_id, + target=final.target, + connection_id=final.connection_id, + status=final.status, + detail=final.detail, + payload_hash=final.payload_hash, + latency_ms=final.latency_ms, + entry_metadata=dict(final.metadata), + redacted=final.redacted, + prev_hash=prev, + entry_hash=entry_hash, + ) + ) + head.last_seq = seq + head.last_hash = entry_hash + session.commit() + return final + + def all(self) -> list[AuditEntry]: + """Every entry, oldest-first (chain order).""" + with self._sf() as session: + rows = ( + session.execute( + select(ObservabilityAuditEntry).order_by( + ObservabilityAuditEntry.seq.asc() + ) + ) + .scalars() + .all() + ) + return [self._to_entry(r) for r in rows] + + def query( + self, + *, + category: AuditCategory | None = None, + actor: str | None = None, + run_id: uuid.UUID | None = None, + connection_id: str | None = None, + workspace_id: uuid.UUID | None = None, + limit: int | None = None, + ) -> list[AuditEntry]: + """Filtered, chain-ordered entries (semantics identical to the in-memory store).""" + stmt = select(ObservabilityAuditEntry) + if category is not None: + stmt = stmt.where(ObservabilityAuditEntry.category == category.value) + if actor is not None: + stmt = stmt.where(ObservabilityAuditEntry.actor == actor) + if run_id is not None: + stmt = stmt.where(ObservabilityAuditEntry.run_id == run_id) + if connection_id is not None: + stmt = stmt.where(ObservabilityAuditEntry.connection_id == connection_id) + if workspace_id is not None: + stmt = stmt.where(ObservabilityAuditEntry.workspace_ref == workspace_id) + + with self._sf() as session: + # ``limit`` mirrors ``rows[-limit:] if limit else []`` for limit>=0: + # the *most recent* N in chain order; 0 → empty; None/negative → all. + if limit is not None and limit >= 0: + if limit == 0: + return [] + rows = list( + session.execute( + stmt.order_by(ObservabilityAuditEntry.seq.desc()).limit(limit) + ) + .scalars() + .all() + ) + rows.reverse() + return [self._to_entry(r) for r in rows] + + rows = ( + session.execute(stmt.order_by(ObservabilityAuditEntry.seq.asc())) + .scalars() + .all() + ) + return [self._to_entry(r) for r in rows] + + def verify_integrity(self) -> bool: + """Re-walk the persisted global chain (same verifier as the in-memory store).""" + return verify_chain(self.all()) + + def count(self) -> int: + """Number of persisted entries (helper; not part of the protocol). + + Deliberately *not* ``__len__``: an empty store must stay truthy so + ``AuditLog(store)`` never mistakes it for "no store" and falls back to an + in-memory one. + """ + with self._sf() as session: + return int( + session.execute( + select(func.count()).select_from(ObservabilityAuditEntry) + ).scalar_one() + ) + + +# --------------------------------------------------------------------------- # +# Composition root # +# --------------------------------------------------------------------------- # + + +def build_audit_store() -> AuditStore: + """Return the process-wide audit store selected by ``FORGE_AUDIT_BACKEND``. + + ``memory`` (default) → the hermetic :class:`InMemoryAuditStore` (unit-test + default, no Postgres); ``db`` → the durable :class:`DbAuditStore` bound to the + shared session factory. Both satisfy the same frozen ``AuditStore`` protocol. + """ + from forge_api.settings import get_settings + + if get_settings().audit_backend == "db": + from forge_api.db import get_session_factory + + return DbAuditStore(get_session_factory()) + return InMemoryAuditStore() + + +def default_audit_log() -> AuditLog: + """Return an :class:`AuditLog` over the env-selected store (see ``build_audit_store``).""" + return AuditLog(build_audit_store()) diff --git a/apps/api/forge_api/observability/service.py b/apps/api/forge_api/observability/service.py index 7826fd40..387cba14 100644 --- a/apps/api/forge_api/observability/service.py +++ b/apps/api/forge_api/observability/service.py @@ -12,6 +12,7 @@ from typing import Any from forge_api.observability.audit import AuditCategory, AuditEntry, AuditLog +from forge_api.observability.audit_db import default_audit_log from forge_api.observability.otel import SpanRecorder, get_span_recorder from forge_api.observability.trace import RunTrace, RunTraceAssembler from forge_contracts import AgentRunResult, Step @@ -32,7 +33,10 @@ def __init__( assembler: RunTraceAssembler | None = None, recorder: SpanRecorder | None = None, ) -> None: - self.audit = audit_log or AuditLog() + # Backend chosen by ``FORGE_AUDIT_BACKEND`` (default ``memory`` → the + # hermetic in-memory store, so existing tests stay green untouched; ``db`` + # → the durable Postgres-backed store behind the same protocol). + self.audit = audit_log or default_audit_log() self.assembler = assembler or RunTraceAssembler() self.recorder = recorder or get_span_recorder() self._runs: dict[uuid.UUID, RunTrace] = {} diff --git a/apps/api/forge_api/routers/mcp.py b/apps/api/forge_api/routers/mcp.py index 8d67d764..1bc825c3 100644 --- a/apps/api/forge_api/routers/mcp.py +++ b/apps/api/forge_api/routers/mcp.py @@ -118,9 +118,12 @@ def _mcp_audit_sink() -> object | None: """ if os.environ.get("FORGE_MCP_AUDIT_BACKEND", "memory").strip().lower() != "db": return None - from forge_api.observability import AuditLog, MCPAuditSink + from forge_api.observability import MCPAuditSink + from forge_api.observability.audit_db import default_audit_log - return MCPAuditSink(AuditLog()) + # ``default_audit_log`` selects the store via ``FORGE_AUDIT_BACKEND`` (default + # ``memory``); set it to ``db`` for the entries to land in durable Postgres. + return MCPAuditSink(default_audit_log()) @lru_cache(maxsize=1) diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index 50575cc7..5faa7a09 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -112,6 +112,13 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # frozen ``BoardService`` protocol. Read via ``FORGE_BOARD_BACKEND``. board_backend: str = "memory" + # Observability audit-store backend selection. ``memory`` (default) keeps the + # hermetic, process-memory ``InMemoryAuditStore`` (unit-test default, no + # Postgres); ``db`` wires the Postgres-backed ``DbAuditStore`` behind the same + # frozen ``AuditStore`` protocol so the platform audit trail (and the MCP + # db-path sink) is durably persisted. Read via ``FORGE_AUDIT_BACKEND``. + audit_backend: str = "memory" + # Filesystem root for the spec engine's SDD artifacts (manifests, plans). spec_root: str = "specs" diff --git a/apps/api/tests/test_audit_store_db.py b/apps/api/tests/test_audit_store_db.py new file mode 100644 index 00000000..69bf9541 --- /dev/null +++ b/apps/api/tests/test_audit_store_db.py @@ -0,0 +1,288 @@ +"""Postgres integration tests for :class:`DbAuditStore` (audit-store persistence). + +Exercises the DB-backed observability audit store against a real pgvector +Postgres via the shared ``pg_engine`` fixture (root ``conftest.py``): the +``AuditStore`` protocol end-to-end — append + hash chaining, full round-trip via +``all()``, every ``query`` filter (category / actor / run_id / connection_id / +workspace_id), chain ordering, the ``limit`` edge cases, ``verify_integrity`` on +a clean chain, out-of-band tamper detection, the unique-``seq`` constraint, +durability + a continued global chain across independent store instances, and +structural conformance to the same frozen ``AuditStore`` protocol the in-memory +store implements. Skips cleanly (parked) when no Postgres is reachable; runs +under ``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. + +Each behaviour mirrors ``tests/test_obs_audit.py`` (the in-memory contract), so +both backends are proven to satisfy the same protocol identically. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator + +import pytest +from sqlalchemy import update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.observability.audit import ( + AuditCategory, + AuditEntry, + AuditLog, + AuditStore, + InMemoryAuditStore, + verify_chain, +) +from forge_api.observability.audit_db import DbAuditStore +from forge_db.base import Base +from forge_db.models.observability_audit import ObservabilityAuditEntry + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def store(factory: sessionmaker[Session]) -> DbAuditStore: + return DbAuditStore(factory) + + +def _entry(action: str, **kwargs: object) -> AuditEntry: + return AuditEntry(category=AuditCategory.AGENT_ACTION, action=action, **kwargs) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_db_store_satisfies_audit_store_protocol(store: DbAuditStore) -> None: + assert isinstance(store, AuditStore) + + +# --------------------------------------------------------------------------- # +# Append + hash chain # +# --------------------------------------------------------------------------- # + + +def test_append_assigns_global_monotonic_sequence(store: DbAuditStore) -> None: + a = store.append(_entry("plan")) + b = store.append(_entry("write")) + c = store.append(_entry("approve")) + assert [a.seq, b.seq, c.seq] == [0, 1, 2] + + +def test_entries_are_hash_chained_to_predecessor(store: DbAuditStore) -> None: + first = store.append(_entry("a0")) + second = store.append(_entry("a1")) + assert first.entry_hash + assert second.prev_hash == first.entry_hash + + +def test_round_trip_preserves_all_fields_and_rehashes(store: DbAuditStore) -> None: + run = uuid.uuid4() + ws = uuid.uuid4() + appended = store.append( + AuditEntry( + category=AuditCategory.MCP_CALL, + action="search_docs", + actor="agent-runner", + workspace_id=ws, + run_id=run, + target="search_docs", + connection_id="conn-1", + status="ok", + detail="looked up docs", + payload_hash="abc123", + latency_ms=42, + metadata={"endpoint": "/v1/things"}, + ) + ) + (loaded,) = store.all() + assert loaded == appended # frozen model equality: every field round-trips + assert loaded.workspace_id == ws + assert loaded.run_id == run + assert loaded.connection_id == "conn-1" + assert loaded.latency_ms == 42 + assert loaded.metadata == {"endpoint": "/v1/things"} + # The persisted hash re-verifies against the reconstructed entry. + assert verify_chain(store.all()) is True + + +def test_workspace_id_is_optional(store: DbAuditStore) -> None: + entry = store.append(_entry("boot", workspace_id=None)) + assert entry.workspace_id is None + assert store.all()[0].workspace_id is None + + +# --------------------------------------------------------------------------- # +# Query: filtering, ordering, limit # +# --------------------------------------------------------------------------- # + + +def test_query_filters_by_category_and_run_id(store: DbAuditStore) -> None: + run = uuid.uuid4() + store.append(AuditEntry(category=AuditCategory.AGENT_ACTION, action="plan", run_id=run)) + store.append(AuditEntry(category=AuditCategory.TOOL_CALL, action="write", run_id=run)) + store.append( + AuditEntry(category=AuditCategory.TOOL_CALL, action="write", run_id=uuid.uuid4()) + ) + + assert len(store.query(run_id=run)) == 2 + assert len(store.query(category=AuditCategory.TOOL_CALL)) == 2 + assert len(store.query(category=AuditCategory.TOOL_CALL, run_id=run)) == 1 + + +def test_query_filters_by_actor_connection_and_workspace(store: DbAuditStore) -> None: + ws = uuid.uuid4() + store.append(_entry("a", actor="alice", workspace_id=ws)) + store.append(_entry("b", actor="bob", connection_id="conn-9")) + store.append(_entry("c", actor="alice")) + + assert [e.action for e in store.query(actor="alice")] == ["a", "c"] + assert [e.action for e in store.query(connection_id="conn-9")] == ["b"] + assert [e.action for e in store.query(workspace_id=ws)] == ["a"] + + +def test_query_returns_chain_order(store: DbAuditStore) -> None: + for i in range(5): + store.append(_entry(f"step-{i}")) + assert [e.action for e in store.query()] == [f"step-{i}" for i in range(5)] + assert [e.seq for e in store.all()] == [0, 1, 2, 3, 4] + + +def test_query_limit_returns_most_recent(store: DbAuditStore) -> None: + for i in range(5): + store.append(_entry(f"step-{i}")) + recent = store.query(limit=2) + assert [e.action for e in recent] == ["step-3", "step-4"] + + +def test_query_limit_zero_is_empty_and_negative_is_ignored(store: DbAuditStore) -> None: + for i in range(3): + store.append(_entry(f"s{i}")) + assert store.query(limit=0) == [] + # Negative limit mirrors the in-memory store: the slice is skipped -> all rows. + assert len(store.query(limit=-1)) == 3 + + +# --------------------------------------------------------------------------- # +# Integrity + tamper detection # +# --------------------------------------------------------------------------- # + + +def test_clean_chain_verifies(store: DbAuditStore) -> None: + for i in range(4): + store.append(_entry(f"a{i}")) + assert store.verify_integrity() is True + + +def test_out_of_band_tampering_breaks_verification( + store: DbAuditStore, factory: sessionmaker[Session] +) -> None: + for i in range(3): + store.append(_entry(f"a{i}")) + assert store.verify_integrity() is True + + # Mutate a persisted row's content out-of-band (the repository exposes no + # such path); the hash chain must detect it. + with factory() as session: + session.execute( + update(ObservabilityAuditEntry) + .where(ObservabilityAuditEntry.seq == 1) + .values(action="MALICIOUS") + ) + session.commit() + + assert store.verify_integrity() is False + + +# --------------------------------------------------------------------------- # +# Constraints + durability # +# --------------------------------------------------------------------------- # + + +def test_duplicate_seq_is_rejected( + store: DbAuditStore, factory: sessionmaker[Session] +) -> None: + store.append(_entry("only")) # seq 0 + with factory() as session: # noqa: SIM117 - explicit raises block + with pytest.raises(IntegrityError): + session.add( + ObservabilityAuditEntry( + entry_id=uuid.uuid4(), + seq=0, # collides with the existing chain position + occurred_at=store.all()[0].timestamp, + category="agent_action", + action="dupe", + status="ok", + prev_hash="0" * 64, + entry_hash="1" * 64, + ) + ) + session.commit() + + +def test_chain_persists_and_continues_across_store_instances( + factory: sessionmaker[Session], +) -> None: + first = DbAuditStore(factory) + first.append(_entry("a0")) + a1 = first.append(_entry("a1")) + + # A brand-new instance sees the durable trail and continues the same chain. + second = DbAuditStore(factory) + assert [e.seq for e in second.all()] == [0, 1] + appended = second.append(_entry("a2")) + assert appended.seq == 2 + assert appended.prev_hash == a1.entry_hash + assert second.verify_integrity() is True + + +# --------------------------------------------------------------------------- # +# Parity with the in-memory store (same protocol, identical behaviour) # +# --------------------------------------------------------------------------- # + + +def test_matches_in_memory_store_behaviour(store: DbAuditStore) -> None: + mem = InMemoryAuditStore() + run = uuid.uuid4() + payloads = [ + _entry("plan", run_id=run), + AuditEntry(category=AuditCategory.TOOL_CALL, action="write", run_id=run, actor="x"), + AuditEntry(category=AuditCategory.APPROVAL, action="approve", actor="x"), + ] + for payload in payloads: + # Same input entry into both stores. + store.append(payload.model_copy()) + mem.append(payload.model_copy()) + + assert [e.seq for e in store.all()] == [e.seq for e in mem.all()] + assert len(store.query(run_id=run)) == len(mem.query(run_id=run)) + assert len(store.query(actor="x")) == len(mem.query(actor="x")) + assert [e.action for e in store.query(limit=2)] == [e.action for e in mem.query(limit=2)] + assert store.verify_integrity() is mem.verify_integrity() is True + + +def test_audit_log_facade_persists_through_db_store(store: DbAuditStore) -> None: + """The redacting :class:`AuditLog` facade writes through the durable store.""" + log = AuditLog(store) + entry = log.record( + category=AuditCategory.TOOL_CALL, + action="call_api", + detail="used Authorization: Bearer abcDEF123456ghiJKL", + metadata={"api_key": "sk-SECRET1234567890", "endpoint": "/v1/things"}, + ) + assert entry.redacted is True + (persisted,) = store.all() + assert persisted == entry + assert "sk-SECRET1234567890" not in persisted.model_dump_json() + assert "abcDEF123456ghiJKL" not in (persisted.detail or "") + assert log.verify_integrity() is True diff --git a/packages/db/forge_db/models/__init__.py b/packages/db/forge_db/models/__init__.py index c5da6b51..c45f20d1 100644 --- a/packages/db/forge_db/models/__init__.py +++ b/packages/db/forge_db/models/__init__.py @@ -106,6 +106,10 @@ ) from forge_db.models.multi_repo import AgentRepoWorkspace, PRGroup from forge_db.models.oauth_account import OAuthAccount +from forge_db.models.observability_audit import ( + ObservabilityAuditChainHead, + ObservabilityAuditEntry, +) from forge_db.models.planning import ( Epic, Incident, @@ -221,6 +225,8 @@ "ModelPrice", "OAuthAccount", "OAuthProvider", + "ObservabilityAuditChainHead", + "ObservabilityAuditEntry", "PMAuthType", "PMConflictPolicy", "PMConnection", diff --git a/packages/db/forge_db/models/observability_audit.py b/packages/db/forge_db/models/observability_audit.py new file mode 100644 index 00000000..6caeac49 --- /dev/null +++ b/packages/db/forge_db/models/observability_audit.py @@ -0,0 +1,127 @@ +"""Durable backing table for the observability audit store (audit-store persist). + +The API's :class:`forge_api.observability.audit.InMemoryAuditStore` keeps a +*global*, append-only, tamper-evident hash chain of :class:`AuditEntry` records +(``category`` / ``actor`` / ``run_id`` / ``connection_id`` / ``workspace_id`` / +``seq`` / ``prev_hash`` / ``entry_hash``). This module is the Postgres backing +for the *db* variant of that store: one row per appended entry plus a single +global cursor row that serializes appends and hands out the next ``seq`` / +``prev_hash``. + +Why a **new** table rather than reusing F39's ``audit_log``: the two are +genuinely different sinks. F39's ``audit_log`` is a *per-workspace* chain of +``AuditEvent`` rows (``workspace_id`` NOT NULL + FK, ``actor_id`` a real +``app_user`` FK, 1-based per-workspace ``seq``, and an ``entry_hash`` computed +over the F39 field tuple). The observability ``AuditEntry`` is a *global* chain +(0-based ``seq`` spanning every workspace), its ``workspace_id`` is optional and +carries no FK, its ``actor`` is a free-form label string, and its ``entry_hash`` +is the SHA-256 of the whole redacted entry model. Forcing one onto the other +would either weaken F39's constraints or change the observability store's +behaviour — so the observability chain gets its own table and keeps byte-for-byte +parity with the in-memory store (the repository re-uses that store's own +``_hash_entry`` helper). + +The entry's logical ``timestamp`` is stored in ``occurred_at`` (timezone-aware); +its ``metadata`` dict lands in JSONB. ``category`` / ``actor`` / ``target`` / +``connection_id`` / ``status`` / ``detail`` are stored in unbounded ``Text`` so +no value is ever truncated (a truncated value would break the hash chain). The +mapping to/from :class:`AuditCategory` lives in the API repository — ``forge_db`` +never imports ``forge_api``. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import BigInteger, Boolean, DateTime, Index, Integer, String, Text, Uuid +from sqlalchemy.orm import Mapped, mapped_column + +from forge_db.base import ForgeModel, json_type + +#: ``prev_hash`` seed of the global chain (mirrors the in-memory store's genesis). +GENESIS_HASH = "0" * 64 + +#: Fixed singleton id of the single global chain-cursor row. +CHAIN_HEAD_ID = uuid.UUID("00000000-0000-0000-0000-0000000a0d17") + + +class ObservabilityAuditEntry(ForgeModel): + """One persisted row of the global observability audit hash chain. + + Not workspace-scoped: ``workspace_id`` is an optional, un-constrained tag + (the in-memory store holds free-floating UUIDs), so this uses the plain + :class:`ForgeModel` (surrogate UUID PK + timestamps) rather than the + tenant-scoped base. ``created_at`` / ``updated_at`` are the DB insert stamps; + ``occurred_at`` is the entry's own logical timestamp. + """ + + __tablename__ = "observability_audit_entry" + __table_args__ = ( + # Global chain integrity: gap-free, unique, monotonic position. + Index("uq_observability_audit_entry_seq", "seq", unique=True), + Index("ix_observability_audit_entry_category", "category"), + Index("ix_observability_audit_entry_actor", "actor"), + Index("ix_observability_audit_entry_run_id", "run_id"), + Index("ix_observability_audit_entry_connection_id", "connection_id"), + Index("ix_observability_audit_entry_workspace_ref", "workspace_ref"), + ) + + #: The :class:`AuditEntry.id` (its own UUID; distinct from the surrogate PK). + entry_id: Mapped[uuid.UUID] = mapped_column(Uuid(as_uuid=True), nullable=False) + #: Global 0-based chain position (unique). + seq: Mapped[int] = mapped_column(BigInteger, nullable=False) + #: The entry's logical timestamp (aware; the in-memory ``AuditEntry.timestamp``). + occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + #: ``AuditCategory`` value (stored as text; mapped in the API repository). + category: Mapped[str] = mapped_column(Text, nullable=False) + action: Mapped[str] = mapped_column(Text, nullable=False) + actor: Mapped[str | None] = mapped_column(Text, nullable=True) + #: Optional, un-constrained tenant tag — a free-floating UUID like the + #: in-memory store holds (accepts any workspace id, existent or not). Named + #: ``workspace_ref`` rather than ``workspace_id`` precisely because it is NOT + #: the tenant FK the house ``workspace_id`` invariant mandates; the repository + #: maps it to/from :attr:`AuditEntry.workspace_id`. + workspace_ref: Mapped[uuid.UUID | None] = mapped_column(Uuid(as_uuid=True), nullable=True) + run_id: Mapped[uuid.UUID | None] = mapped_column(Uuid(as_uuid=True), nullable=True) + target: Mapped[str | None] = mapped_column(Text, nullable=True) + connection_id: Mapped[str | None] = mapped_column(Text, nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, default="ok") + detail: Mapped[str | None] = mapped_column(Text, nullable=True) + payload_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + #: Redacted metadata dict (JSONB); ``metadata`` is reserved on the declarative + #: base, so the attribute + column are named ``entry_metadata``. + entry_metadata: Mapped[dict[str, Any]] = mapped_column( + json_type(), default=dict, nullable=False + ) + redacted: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + #: ``entry_hash`` of the predecessor (``GENESIS_HASH`` for ``seq`` 0). + prev_hash: Mapped[str] = mapped_column(String(64), nullable=False) + #: SHA-256 hex over the canonical redacted entry (in-memory ``_hash_entry``). + entry_hash: Mapped[str] = mapped_column(String(64), nullable=False) + + +class ObservabilityAuditChainHead(ForgeModel): + """The single global chain cursor — the only *mutable* observability-audit row. + + Locked ``FOR UPDATE`` by the repository to serialize appends and hand out the + next ``seq`` / ``prev_hash``. Exactly one row exists, keyed by the fixed + :data:`CHAIN_HEAD_ID` sentinel; ``last_seq`` starts at ``-1`` so the first + append is ``seq`` 0 (parity with the in-memory ``len(entries)`` scheme). + """ + + __tablename__ = "observability_audit_chain_head" + + last_seq: Mapped[int] = mapped_column(BigInteger, default=-1, nullable=False) + last_hash: Mapped[str] = mapped_column(String(64), default=GENESIS_HASH, nullable=False) + + +__all__ = [ + "CHAIN_HEAD_ID", + "GENESIS_HASH", + "ObservabilityAuditChainHead", + "ObservabilityAuditEntry", +] diff --git a/packages/db/migrations/versions/0025_observability_audit_store.py b/packages/db/migrations/versions/0025_observability_audit_store.py new file mode 100644 index 00000000..2c8ddc7f --- /dev/null +++ b/packages/db/migrations/versions/0025_observability_audit_store.py @@ -0,0 +1,73 @@ +"""observability audit store: global hash-chained entry table + chain cursor + +Backs the *db* variant of the API's observability audit store +(``forge_api.observability.audit`` — the platform sink the MCP db-path forwards +to) with real Postgres persistence. Creates two new, self-contained tables: + +* ``observability_audit_entry`` — one append-only row per audit entry, carrying + the global tamper-evident hash chain (``seq`` / ``prev_hash`` / ``entry_hash``) + plus the entry payload (``category`` / ``actor`` / ``run_id`` / + ``connection_id`` / optional ``workspace_id`` / ``metadata`` / ...); +* ``observability_audit_chain_head`` — the single global cursor row that + serializes appends and hands out the next ``seq`` / ``prev_hash``. + +These are distinct from F39's per-workspace ``audit_log`` / ``audit_chain_head`` +(different sink, different chain semantics — see the model module docstring), so +this revision only *adds* tables and touches nothing existing. + +Foundation note (mirrors 0024): ``forge_db``'s metadata is the source of truth, +so a fresh chain already provisions these tables from the models. To stay +idiomatic *and* own an explicit, reversible step this migration is idempotent: +``upgrade`` creates only what is missing, ``downgrade`` drops only what this +revision introduced. Applies cleanly on SQLite (unit path) and pgvector Postgres. + +Revision ID: 0025_observability_audit_store +Revises: 0024_board_persistence +Create Date: 2026-07-05 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) +from forge_db.base import Base + +# revision identifiers, used by Alembic. +revision: str = "0025_observability_audit_store" +down_revision: str | None = "0024_board_persistence" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +# Tables owned by this revision, in create order (no inter-table FK, so order is +# cosmetic; downgrade drops them in reverse). +_TABLES: tuple[str, ...] = ( + "observability_audit_entry", + "observability_audit_chain_head", +) + + +def _owned_tables() -> list[sa.Table]: + by_name = {t.name: t for t in Base.metadata.sorted_tables} + return [by_name[name] for name in _TABLES if name in by_name] + + +def _existing_tables() -> set[str]: + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def upgrade() -> None: + existing = _existing_tables() + to_create = [t for t in _owned_tables() if t.name not in existing] + if to_create: + Base.metadata.create_all(bind=op.get_bind(), tables=to_create) + + +def downgrade() -> None: + existing = _existing_tables() + to_drop = [t for t in reversed(_owned_tables()) if t.name in existing] + if to_drop: + Base.metadata.drop_all(bind=op.get_bind(), tables=to_drop) diff --git a/packages/db/tests/test_models.py b/packages/db/tests/test_models.py index a842c982..b95843c2 100644 --- a/packages/db/tests/test_models.py +++ b/packages/db/tests/test_models.py @@ -135,6 +135,11 @@ # F01 board persistence: task-dependency adjacency (depends_on edges) backing # the DB-backed SqlAlchemyBoardService. "TaskDependency", + # audit-store persistence: durable backing for the observability audit store + # (global hash chain + its single cursor row). Not tenant-scoped — the entry's + # optional workspace tag is a free UUID (``workspace_ref``), never the FK. + "ObservabilityAuditEntry", + "ObservabilityAuditChainHead", ] # Tables that are NOT the tenant root and therefore must carry a workspace FK. @@ -153,6 +158,11 @@ # A frozen benchmark suite is a global community artifact (F35 §3.1); # submissions carry a *nullable* workspace_id (NULL = official/system). "benchmark_suite", + # The observability audit store is a *global* (cross-workspace) hash chain, + # mirroring the in-memory store: the entry carries only an optional, un-FK'd + # ``workspace_ref`` tag, and the cursor row no workspace column at all. + "observability_audit_entry", + "observability_audit_chain_head", } From acfa2d02e631fab7a6f71a0e25792c4c7ef4107e Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 12:07:15 +0200 Subject: [PATCH 03/12] feat(db/approval-repository): Postgres persistence Co-Authored-By: Claude Fable 5 --- .../services/approval_repository_db.py | 286 ++++++++++ .../forge_api/services/approval_service.py | 25 +- apps/api/forge_api/settings.py | 7 + apps/api/tests/test_approval_repository_db.py | 505 ++++++++++++++++++ packages/db/forge_db/models/runs.py | 13 + .../0026_approval_repository_columns.py | 84 +++ packages/db/tests/test_migration.py | 35 ++ 7 files changed, 954 insertions(+), 1 deletion(-) create mode 100644 apps/api/forge_api/services/approval_repository_db.py create mode 100644 apps/api/tests/test_approval_repository_db.py create mode 100644 packages/db/migrations/versions/0026_approval_repository_columns.py diff --git a/apps/api/forge_api/services/approval_repository_db.py b/apps/api/forge_api/services/approval_repository_db.py new file mode 100644 index 00000000..14812303 --- /dev/null +++ b/apps/api/forge_api/services/approval_repository_db.py @@ -0,0 +1,286 @@ +"""Postgres-backed :class:`~forge_approval.repository.ApprovalRepository` (F36). + +:class:`SqlAlchemyApprovalRepository` is a drop-in, durable alternative to +:class:`~forge_approval.repository.InMemoryApprovalRepository` that satisfies the +**same** async ``ApprovalRepository`` protocol (``add`` / ``get`` / +``find_pending`` / ``list`` / ``update`` / ``add_decision`` / ``decisions_for``) +— so the F36 composition root swaps it in behind ``FORGE_APPROVAL_BACKEND=db`` +with no behavioural change. The default stays ``memory`` and the in-memory store +remains the unit-test default. + +It lives in ``apps/api`` (not the ``forge_approval`` SDK, which is deliberately +"pure domain — no DB") exactly like the sibling +:class:`~forge_api.observability.audit_db.DbAuditStore`: the frozen SDK stays +DB-free and this adapter maps the domain :class:`ApprovalRequest` / +:class:`ApprovalDecisionRecord` onto the canonical F36 ORM rows in ``forge_db``. + +Behaviour parity with the in-memory store is exact and intentional: + +* the domain ``ApprovalRequest`` maps onto the baseline ``approval_request`` row + whose F36 columns keep their baseline names (``gate`` / ``summary`` / + ``payload`` / ``decided_by_id`` / ``decided_at`` / ``decision_reason``), with + ``requested_at`` carried by the row's ``created_at`` timestamp and the two + repository-only fields (``requested_actor`` / ``escalated``) added by revision + ``0026``; +* ``add_decision`` raises :class:`DuplicateDecisionError` when the + one-vote-per-approver ``uq_approval_decision_approver`` unique fires (the DB + analogue of the in-memory per-approver guard), and ``update`` raises + :class:`ApprovalNotFoundError` for an unknown id — the exact errors the + service already handles; +* every read is workspace-scoped, so a cross-workspace id reads as absent. + +The one storage-boundary divergence (shared with every DB-backed repo here) is +that the DB's ``uq_pending_gate`` partial-unique and the ``approval_decision`` +foreign key are *real*: the service always calls ``find_pending`` before ``add`` +and loads the parent before ``add_decision``, so these never surface in normal +flow, but a direct duplicate insert raises at the storage boundary rather than +silently succeeding. +""" + +from __future__ import annotations + +import builtins +import uuid +from datetime import UTC, datetime +from typing import TYPE_CHECKING, cast + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from forge_approval.models import ( + ApprovalAction, + ApprovalDecisionRecord, + ApprovalRequest, + GateStatus, + GateType, + RiskLevel, +) +from forge_approval.repository import ( + ApprovalNotFoundError, + DuplicateDecisionError, +) +from forge_db.models import ApprovalDecision as ApprovalDecisionRow +from forge_db.models import ApprovalRequest as ApprovalRequestRow +from forge_db.models.enums import ApprovalStatus + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["SqlAlchemyApprovalRepository"] + + +def _aware(value: datetime | None) -> datetime | None: + """Normalise a stored timestamp to timezone-aware UTC (SQLite reads naive).""" + if value is None: + return None + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +def _db_status(status: GateStatus) -> ApprovalStatus: + """Domain gate status -> the DB ``ApprovalStatus`` (value-identical enums).""" + return ApprovalStatus(status.value) + + +def _gate_status(value: object) -> GateStatus: + """DB ``ApprovalStatus`` (or raw string) -> the domain :class:`GateStatus`.""" + return GateStatus(value.value if isinstance(value, ApprovalStatus) else str(value)) + + +class SqlAlchemyApprovalRepository: + """A Postgres-backed approval repository (implements ``ApprovalRepository``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + # ------------------------------------------------------------------ # + # Mapping # + # ------------------------------------------------------------------ # + + def _apply(self, row: ApprovalRequestRow, request: ApprovalRequest) -> None: + """Write every domain field onto ``row`` (used by ``add`` + ``update``).""" + row.workspace_id = request.workspace_id + row.project_id = request.project_id + row.gate = request.gate_type + row.status = _db_status(request.status) + row.subject_type = request.subject_type + row.subject_id = request.subject_id + row.workflow_run_id = request.workflow_run_id + row.agent_run_id = request.agent_run_id + row.task_id = request.task_id + row.required_approvals = request.required_approvals + row.risk_level = request.risk_level + row.summary = request.title + row.payload = dict(request.gate_payload or {}) + row.context_ref = request.context_ref + row.requested_by = str(request.requested_by) if request.requested_by else None + row.requested_actor = request.requested_actor + row.escalated = request.escalated + row.decision_reason = request.decision_note + row.decided_by_id = request.resolver_user_id + row.expires_at = request.expires_at + row.decided_at = request.resolved_at + # ``requested_at`` is carried by the row's ``created_at`` timestamp column. + if request.requested_at is not None: + row.created_at = request.requested_at + + def _to_domain(self, row: ApprovalRequestRow) -> ApprovalRequest: + """Rebuild the domain :class:`ApprovalRequest` from a persisted row.""" + return ApprovalRequest( + id=row.id, + workspace_id=row.workspace_id, + project_id=row.project_id, + gate_type=GateType(row.gate.value if hasattr(row.gate, "value") else row.gate), + status=_gate_status(row.status), + subject_type=row.subject_type or "workflow_run", + subject_id=row.subject_id, + workflow_run_id=row.workflow_run_id, + agent_run_id=row.agent_run_id, + task_id=row.task_id, + required_approvals=row.required_approvals, + risk_level=cast(RiskLevel, row.risk_level), + title=row.summary, + gate_payload=dict(row.payload or {}), + context_ref=row.context_ref, + requested_by=uuid.UUID(row.requested_by) if row.requested_by else None, + requested_actor=row.requested_actor, + escalated=row.escalated, + decision_note=row.decision_reason, + resolver_user_id=row.decided_by_id, + expires_at=_aware(row.expires_at), + requested_at=_aware(row.created_at), + resolved_at=_aware(row.decided_at), + ) + + def _decision_to_domain(self, row: ApprovalDecisionRow) -> ApprovalDecisionRecord: + return ApprovalDecisionRecord( + approval_request_id=row.approval_request_id, + approver_user_id=row.approver_user_id, + decision=ApprovalAction(row.decision), + note=row.note, + created_at=_aware(row.created_at), + ) + + # ------------------------------------------------------------------ # + # ApprovalRepository protocol # + # ------------------------------------------------------------------ # + + async def add(self, request: ApprovalRequest) -> ApprovalRequest: + stored = request.model_copy(deep=True) + if stored.requested_at is None: + stored.requested_at = datetime.now(UTC) + with self._sf() as session: + row = ApprovalRequestRow(id=stored.id) + self._apply(row, stored) + session.add(row) + session.commit() + return stored.model_copy(deep=True) + + async def get( + self, approval_id: uuid.UUID, *, workspace_id: uuid.UUID + ) -> ApprovalRequest | None: + with self._sf() as session: + row = session.get(ApprovalRequestRow, approval_id) + if row is None or row.workspace_id != workspace_id: + return None + return self._to_domain(row) + + async def find_pending( + self, + *, + workspace_id: uuid.UUID, + subject_type: str, + subject_id: uuid.UUID | None, + gate_type: GateType, + ) -> ApprovalRequest | None: + if subject_id is None: + return None + with self._sf() as session: + row = session.scalars( + select(ApprovalRequestRow) + .where( + ApprovalRequestRow.workspace_id == workspace_id, + ApprovalRequestRow.status == _db_status(GateStatus.PENDING), + ApprovalRequestRow.subject_type == subject_type, + ApprovalRequestRow.subject_id == subject_id, + ApprovalRequestRow.gate == gate_type, + ) + .limit(1) + ).first() + return self._to_domain(row) if row is not None else None + + async def list( + self, + *, + workspace_id: uuid.UUID, + status: GateStatus | None = None, + gate_type: GateType | None = None, + project_id: uuid.UUID | None = None, + ) -> builtins.list[ApprovalRequest]: + with self._sf() as session: + stmt = select(ApprovalRequestRow).where( + ApprovalRequestRow.workspace_id == workspace_id + ) + if status is not None: + stmt = stmt.where(ApprovalRequestRow.status == _db_status(status)) + if gate_type is not None: + stmt = stmt.where(ApprovalRequestRow.gate == gate_type) + if project_id is not None: + stmt = stmt.where(ApprovalRequestRow.project_id == project_id) + # Deterministic insertion order (``created_at`` == ``requested_at``); + # the service re-sorts the inbox by risk afterwards. + stmt = stmt.order_by( + ApprovalRequestRow.created_at.asc(), ApprovalRequestRow.id.asc() + ) + rows = session.scalars(stmt).all() + return [self._to_domain(r) for r in rows] + + async def update(self, request: ApprovalRequest) -> ApprovalRequest: + with self._sf() as session: + row = session.get(ApprovalRequestRow, request.id) + if row is None: + raise ApprovalNotFoundError(request.id) + self._apply(row, request) + session.commit() + return request.model_copy(deep=True) + + async def add_decision( + self, record: ApprovalDecisionRecord + ) -> ApprovalDecisionRecord: + stored = record.model_copy(deep=True) + if stored.created_at is None: + stored.created_at = datetime.now(UTC) + with self._sf() as session: + parent = session.get(ApprovalRequestRow, stored.approval_request_id) + if parent is None: + raise ApprovalNotFoundError(stored.approval_request_id) + session.add( + ApprovalDecisionRow( + workspace_id=parent.workspace_id, + approval_request_id=stored.approval_request_id, + approver_user_id=stored.approver_user_id, + decision=stored.decision.value, + note=stored.note, + created_at=stored.created_at, + ) + ) + try: + session.commit() + except IntegrityError as exc: # one-vote-per-approver unique + session.rollback() + raise DuplicateDecisionError( + stored.approval_request_id, stored.approver_user_id + ) from exc + return stored.model_copy(deep=True) + + async def decisions_for( + self, approval_id: uuid.UUID + ) -> builtins.list[ApprovalDecisionRecord]: + with self._sf() as session: + rows = session.scalars( + select(ApprovalDecisionRow) + .where(ApprovalDecisionRow.approval_request_id == approval_id) + .order_by( + ApprovalDecisionRow.created_at.asc(), ApprovalDecisionRow.id.asc() + ) + ).all() + return [self._decision_to_domain(r) for r in rows] diff --git a/apps/api/forge_api/services/approval_service.py b/apps/api/forge_api/services/approval_service.py index 08131ee2..f8549ea6 100644 --- a/apps/api/forge_api/services/approval_service.py +++ b/apps/api/forge_api/services/approval_service.py @@ -27,6 +27,7 @@ from forge_api.observability.service import get_observability_service from forge_approval import ( ApprovalAuthorizer, + ApprovalRepository, ApprovalService, GateRegistry, InMemoryActivityBus, @@ -66,11 +67,32 @@ def get_gate_registry() -> GateRegistry: return build_gate_registry(get_override_grant_store()) +def build_approval_repository() -> ApprovalRepository: + """Return the approval repository selected by ``FORGE_APPROVAL_BACKEND``. + + ``memory`` (default) -> the hermetic :class:`InMemoryApprovalRepository` + (unit-test default, no Postgres); ``db`` -> the durable + :class:`~forge_api.services.approval_repository_db.SqlAlchemyApprovalRepository` + bound to the shared session factory. Both satisfy the same async + ``ApprovalRepository`` protocol, so the swap is behaviour-preserving. + """ + from forge_api.settings import get_settings + + if get_settings().approval_backend == "db": + from forge_api.db import get_session_factory + from forge_api.services.approval_repository_db import ( + SqlAlchemyApprovalRepository, + ) + + return SqlAlchemyApprovalRepository(get_session_factory()) + return InMemoryApprovalRepository() + + @lru_cache(maxsize=1) def get_approval_service() -> ApprovalService: """Process-wide unified approval service (override in tests via DI).""" return ApprovalService( - InMemoryApprovalRepository(), + build_approval_repository(), get_gate_registry(), ApprovalAuthorizer(), events=InMemoryActivityBus(), @@ -97,6 +119,7 @@ def to_approval_principal(principal: Principal) -> ApprovalPrincipal: __all__ = [ + "build_approval_repository", "build_gate_registry", "get_approval_service", "get_gate_registry", diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index 5faa7a09..dc59a5f4 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -119,6 +119,13 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # db-path sink) is durably persisted. Read via ``FORGE_AUDIT_BACKEND``. audit_backend: str = "memory" + # F36 approval-repository backend selection. ``memory`` (default) keeps the + # hermetic, process-memory ``InMemoryApprovalRepository`` (unit-test default, + # no Postgres); ``db`` wires the Postgres-backed ``SqlAlchemyApprovalRepository`` + # behind the same ``ApprovalRepository`` protocol so approval gates + decisions + # are durably persisted. Read via ``FORGE_APPROVAL_BACKEND``. + approval_backend: str = "memory" + # Filesystem root for the spec engine's SDD artifacts (manifests, plans). spec_root: str = "specs" diff --git a/apps/api/tests/test_approval_repository_db.py b/apps/api/tests/test_approval_repository_db.py new file mode 100644 index 00000000..3f142796 --- /dev/null +++ b/apps/api/tests/test_approval_repository_db.py @@ -0,0 +1,505 @@ +"""Postgres integration tests for :class:`SqlAlchemyApprovalRepository` (F36). + +Exercises the DB-backed approval repository against a real pgvector Postgres via +the shared ``pg_engine`` fixture (root ``conftest.py``): the async +``ApprovalRepository`` protocol end-to-end — a full ``ApprovalRequest`` round-trip +(every domain field, including the repository-only ``requested_actor`` / +``escalated`` and the ``gate_payload`` JSONB), workspace-scoped reads + +cross-workspace isolation, ``find_pending`` semantics, ``list`` filtering + +ordering, ``update`` (status transition, escalation flag, unknown-id +``ApprovalNotFoundError``), the append-only per-approver decision trail +(``add_decision`` + ``DuplicateDecisionError`` + ``decisions_for`` ordering), the +``uq_pending_gate`` storage-boundary constraint, durability across repository +instances, and structural conformance to the same protocol the in-memory store +implements. Skips cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. + +Each behaviour mirrors the in-memory contract, so both backends are proven to +satisfy the same protocol identically. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.services.approval_repository_db import SqlAlchemyApprovalRepository +from forge_approval.models import ( + ApprovalAction, + ApprovalDecisionRecord, + ApprovalRequest, + GateStatus, + GateType, +) +from forge_approval.repository import ( + ApprovalNotFoundError, + ApprovalRepository, + DuplicateDecisionError, + InMemoryApprovalRepository, +) +from forge_db.base import Base +from forge_db.models import User, Workspace + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def seed(factory: sessionmaker[Session]) -> dict[str, uuid.UUID]: + """A workspace + two reviewers, plus a second isolated workspace.""" + ws = uuid.uuid4() + other_ws = uuid.uuid4() + alice = uuid.uuid4() + bob = uuid.uuid4() + with factory() as session: + session.add(Workspace(id=ws, name="Acme", slug=f"acme-{uuid.uuid4().hex[:8]}")) + session.add( + Workspace(id=other_ws, name="Other", slug=f"other-{uuid.uuid4().hex[:8]}") + ) + session.flush() + session.add( + User(id=alice, workspace_id=ws, email=f"a-{alice.hex[:6]}@acme.dev", name="Alice") + ) + session.add( + User(id=bob, workspace_id=ws, email=f"b-{bob.hex[:6]}@acme.dev", name="Bob") + ) + session.commit() + return {"ws": ws, "other_ws": other_ws, "alice": alice, "bob": bob} + + +@pytest.fixture +def repo(factory: sessionmaker[Session]) -> SqlAlchemyApprovalRepository: + return SqlAlchemyApprovalRepository(factory) + + +def _request( + ws: uuid.UUID, + *, + gate: GateType = GateType.PR, + status: GateStatus = GateStatus.PENDING, + subject_id: uuid.UUID | None = None, + project_id: uuid.UUID | None = None, + risk_level: str = "info", + requested_actor: str = "system", + requested_at: datetime | None = None, + **kw, +) -> ApprovalRequest: + return ApprovalRequest( + id=uuid.uuid4(), + workspace_id=ws, + project_id=project_id, + gate_type=gate, + status=status, + subject_type="workflow_run", + subject_id=subject_id or uuid.uuid4(), + risk_level=risk_level, # type: ignore[arg-type] + requested_actor=requested_actor, + requested_at=requested_at or datetime.now(UTC), + **kw, + ) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_repo_satisfies_approval_repository_protocol( + repo: SqlAlchemyApprovalRepository, +) -> None: + assert isinstance(repo, ApprovalRepository) + + +# --------------------------------------------------------------------------- # +# add + get round-trip # +# --------------------------------------------------------------------------- # + + +async def test_add_then_get_round_trips_every_field( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + requested_at = datetime(2026, 7, 5, 12, 0, 0, 123456, tzinfo=UTC) + expires_at = datetime(2026, 7, 5, 16, 0, 0, 654321, tzinfo=UTC) + request = _request( + ws, + gate=GateType.DEPLOY, + risk_level="critical", + requested_actor=f"user:{seed['alice']}", + requested_at=requested_at, + project_id=uuid.uuid4(), + required_approvals=2, + title="Ship it", + gate_payload={"env": "prod", "nested": {"k": [1, 2, 3]}}, + context_ref="s3://ctx/abc", + requested_by=seed["alice"], + escalated=True, + expires_at=expires_at, + ) + returned = await repo.add(request) + # ``add`` returns the stored request verbatim (parity with the in-memory store). + assert returned.id == request.id + assert returned.requested_actor == f"user:{seed['alice']}" + + loaded = await repo.get(request.id, workspace_id=ws) + assert loaded is not None + assert loaded.id == request.id + assert loaded.gate_type is GateType.DEPLOY + assert loaded.status is GateStatus.PENDING + assert loaded.subject_type == "workflow_run" + assert loaded.subject_id == request.subject_id + assert loaded.risk_level == "critical" + assert loaded.required_approvals == 2 + assert loaded.title == "Ship it" + assert loaded.gate_payload == {"env": "prod", "nested": {"k": [1, 2, 3]}} + assert loaded.context_ref == "s3://ctx/abc" + assert loaded.requested_by == seed["alice"] + assert loaded.requested_actor == f"user:{seed['alice']}" + assert loaded.escalated is True + assert loaded.project_id == request.project_id + assert loaded.requested_at == requested_at + assert loaded.expires_at == expires_at + + +async def test_add_fills_requested_at_when_missing( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + request = _request(seed["ws"]) + request.requested_at = None + returned = await repo.add(request) + assert returned.requested_at is not None + loaded = await repo.get(request.id, workspace_id=seed["ws"]) + assert loaded is not None and loaded.requested_at is not None + + +async def test_get_is_workspace_scoped( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + request = _request(seed["ws"]) + await repo.add(request) + # Correct workspace: found. Foreign workspace: reads as absent (never a leak). + assert await repo.get(request.id, workspace_id=seed["ws"]) is not None + assert await repo.get(request.id, workspace_id=seed["other_ws"]) is None + assert await repo.get(uuid.uuid4(), workspace_id=seed["ws"]) is None + + +# --------------------------------------------------------------------------- # +# find_pending # +# --------------------------------------------------------------------------- # + + +async def test_find_pending_matches_subject_gate_and_status( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + subject = uuid.uuid4() + request = _request(ws, gate=GateType.PR, subject_id=subject) + await repo.add(request) + + found = await repo.find_pending( + workspace_id=ws, + subject_type="workflow_run", + subject_id=subject, + gate_type=GateType.PR, + ) + assert found is not None and found.id == request.id + + # Different gate type / subject / workspace -> no match. + assert ( + await repo.find_pending( + workspace_id=ws, + subject_type="workflow_run", + subject_id=subject, + gate_type=GateType.SPEC, + ) + is None + ) + assert ( + await repo.find_pending( + workspace_id=seed["other_ws"], + subject_type="workflow_run", + subject_id=subject, + gate_type=GateType.PR, + ) + is None + ) + # A None subject id never matches (mirrors the in-memory guard). + assert ( + await repo.find_pending( + workspace_id=ws, + subject_type="workflow_run", + subject_id=None, + gate_type=GateType.PR, + ) + is None + ) + + +async def test_find_pending_ignores_resolved_gates( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + subject = uuid.uuid4() + request = _request(ws, subject_id=subject) + await repo.add(request) + request.status = GateStatus.APPROVED + await repo.update(request) + assert ( + await repo.find_pending( + workspace_id=ws, + subject_type="workflow_run", + subject_id=subject, + gate_type=GateType.PR, + ) + is None + ) + + +# --------------------------------------------------------------------------- # +# list: filtering + ordering # +# --------------------------------------------------------------------------- # + + +async def test_list_filters_and_orders_by_requested_at( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + proj = uuid.uuid4() + base = datetime(2026, 7, 5, 9, 0, 0, tzinfo=UTC) + r1 = _request(ws, gate=GateType.PR, project_id=proj, requested_at=base) + r2 = _request( + ws, gate=GateType.SPEC, requested_at=base + timedelta(minutes=1) + ) + r3 = _request( + ws, gate=GateType.PR, project_id=proj, requested_at=base + timedelta(minutes=2) + ) + r3.status = GateStatus.APPROVED + for r in (r2, r3, r1): # insert out of order; list must sort by requested_at + await repo.add(r) + # Foreign-workspace row must never appear. + await repo.add(_request(seed["other_ws"])) + + all_ws = await repo.list(workspace_id=ws) + assert [r.id for r in all_ws] == [r1.id, r2.id, r3.id] + + assert [r.id for r in await repo.list(workspace_id=ws, status=GateStatus.PENDING)] == [ + r1.id, + r2.id, + ] + assert [r.id for r in await repo.list(workspace_id=ws, gate_type=GateType.PR)] == [ + r1.id, + r3.id, + ] + assert [r.id for r in await repo.list(workspace_id=ws, project_id=proj)] == [ + r1.id, + r3.id, + ] + assert ( + await repo.list(workspace_id=ws, gate_type=GateType.PR, status=GateStatus.PENDING) + )[0].id == r1.id + assert await repo.list(workspace_id=uuid.uuid4()) == [] + + +# --------------------------------------------------------------------------- # +# update # +# --------------------------------------------------------------------------- # + + +async def test_update_persists_resolution( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + request = _request(ws) + await repo.add(request) + + resolved_at = datetime(2026, 7, 5, 13, 0, 0, tzinfo=UTC) + request.status = GateStatus.REJECTED + request.resolver_user_id = seed["bob"] + request.decision_note = "no thanks" + request.resolved_at = resolved_at + returned = await repo.update(request) + assert returned.status is GateStatus.REJECTED + + loaded = await repo.get(request.id, workspace_id=ws) + assert loaded is not None + assert loaded.status is GateStatus.REJECTED + assert loaded.resolver_user_id == seed["bob"] + assert loaded.decision_note == "no thanks" + assert loaded.resolved_at == resolved_at + + +async def test_update_persists_escalation_flag( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + request = _request(ws, risk_level="info") + await repo.add(request) + request.escalated = True + request.risk_level = "critical" # type: ignore[assignment] + await repo.update(request) + loaded = await repo.get(request.id, workspace_id=ws) + assert loaded is not None + assert loaded.escalated is True + assert loaded.risk_level == "critical" + + +async def test_update_unknown_id_raises_not_found( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ghost = _request(seed["ws"]) # never added + with pytest.raises(ApprovalNotFoundError): + await repo.update(ghost) + + +# --------------------------------------------------------------------------- # +# decisions: append-only, one vote per approver # +# --------------------------------------------------------------------------- # + + +async def test_add_decision_and_decisions_for_ordering( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + request = _request(ws) + await repo.add(request) + + first = await repo.add_decision( + ApprovalDecisionRecord( + approval_request_id=request.id, + approver_user_id=seed["alice"], + decision=ApprovalAction.APPROVE, + note="lgtm", + created_at=datetime(2026, 7, 5, 10, 0, 0, tzinfo=UTC), + ) + ) + assert first.created_at is not None + second = await repo.add_decision( + ApprovalDecisionRecord( + approval_request_id=request.id, + approver_user_id=seed["bob"], + decision=ApprovalAction.REQUEST_CHANGES, + created_at=datetime(2026, 7, 5, 10, 5, 0, tzinfo=UTC), + ) + ) + + records = await repo.decisions_for(request.id) + assert [r.approver_user_id for r in records] == [seed["alice"], seed["bob"]] + assert records[0].decision is ApprovalAction.APPROVE + assert records[0].note == "lgtm" + assert records[1].decision is ApprovalAction.REQUEST_CHANGES + assert records[1].note is None + assert second.approver_user_id == seed["bob"] + + +async def test_add_decision_duplicate_approver_raises( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + request = _request(ws) + await repo.add(request) + + vote = ApprovalDecisionRecord( + approval_request_id=request.id, + approver_user_id=seed["alice"], + decision=ApprovalAction.APPROVE, + ) + await repo.add_decision(vote) + with pytest.raises(DuplicateDecisionError): + await repo.add_decision( + ApprovalDecisionRecord( + approval_request_id=request.id, + approver_user_id=seed["alice"], + decision=ApprovalAction.REJECT, + ) + ) + # The append-only trail is unchanged after the rejected duplicate. + assert len(await repo.decisions_for(request.id)) == 1 + + +async def test_decisions_for_unknown_request_is_empty( + repo: SqlAlchemyApprovalRepository, +) -> None: + assert await repo.decisions_for(uuid.uuid4()) == [] + + +# --------------------------------------------------------------------------- # +# Storage-boundary constraint + durability # +# --------------------------------------------------------------------------- # + + +async def test_pending_unique_blocks_duplicate_open_gate( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + """The DB ``uq_pending_gate`` guards against a second open gate per subject. + + The service dedupes via ``find_pending`` first; a *direct* duplicate insert + raises at the storage boundary rather than silently succeeding. + """ + ws = seed["ws"] + subject = uuid.uuid4() + await repo.add(_request(ws, gate=GateType.PR, subject_id=subject)) + with pytest.raises(IntegrityError): + await repo.add(_request(ws, gate=GateType.PR, subject_id=subject)) + + +async def test_persists_across_repository_instances( + factory: sessionmaker[Session], seed: dict[str, uuid.UUID] +) -> None: + first = SqlAlchemyApprovalRepository(factory) + request = _request(seed["ws"], title="durable") + await first.add(request) + + second = SqlAlchemyApprovalRepository(factory) + loaded = await second.get(request.id, workspace_id=seed["ws"]) + assert loaded is not None and loaded.title == "durable" + + +# --------------------------------------------------------------------------- # +# Parity with the in-memory store (same protocol, identical behaviour) # +# --------------------------------------------------------------------------- # + + +async def test_matches_in_memory_store_behaviour( + repo: SqlAlchemyApprovalRepository, seed: dict[str, uuid.UUID] +) -> None: + ws = seed["ws"] + mem: InMemoryApprovalRepository = InMemoryApprovalRepository() + + reqs = [ + _request(ws, gate=GateType.PR, risk_level="warning"), + _request(ws, gate=GateType.SPEC, risk_level="info"), + ] + for r in reqs: + await repo.add(r.model_copy(deep=True)) + await mem.add(r.model_copy(deep=True)) + + db_pending = await repo.list(workspace_id=ws, status=GateStatus.PENDING) + mem_pending = await mem.list(workspace_id=ws, status=GateStatus.PENDING) + assert {r.id for r in db_pending} == {r.id for r in mem_pending} + assert {r.id for r in await repo.list(workspace_id=ws, gate_type=GateType.PR)} == { + r.id for r in await mem.list(workspace_id=ws, gate_type=GateType.PR) + } + + # find_pending agrees on both backends. + subject = reqs[0].subject_id + db_found = await repo.find_pending( + workspace_id=ws, subject_type="workflow_run", subject_id=subject, gate_type=GateType.PR + ) + mem_found = await mem.find_pending( + workspace_id=ws, subject_type="workflow_run", subject_id=subject, gate_type=GateType.PR + ) + assert (db_found is None) == (mem_found is None) + assert db_found is not None and db_found.id == mem_found.id diff --git a/packages/db/forge_db/models/runs.py b/packages/db/forge_db/models/runs.py index 02d94971..007f852b 100644 --- a/packages/db/forge_db/models/runs.py +++ b/packages/db/forge_db/models/runs.py @@ -234,6 +234,19 @@ class ApprovalRequest(WorkspaceScopedModel): decision_reason: Mapped[str | None] = mapped_column(Text, nullable=True) # F36 — optional SLA; the worker sweeper marks overdue pending gates expired. expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + # F36-persist — the requesting actor reference ("system" | ":"). + # Distinct from ``requested_by`` (the resolvable user id): it is what the inbox + # shows and ``approval.requested`` carries, so the DB-backed repository must + # round-trip it verbatim. Server-defaulted so pre-existing rows read "system". + requested_actor: Mapped[str] = mapped_column( + String(64), default="system", server_default=text("'system'"), nullable=False + ) + # F36-persist — escalation raises the resolving bar to admin. Persisted on the + # parent row so the single authorizer re-enforces the admin-only rule on every + # subsequent resolve attempt *after a reload* (not just in-process). + escalated: Mapped[bool] = mapped_column( + Boolean(), default=False, server_default=text("false"), nullable=False + ) agent_run: Mapped[AgentRun | None] = relationship(back_populates="approval_requests") decisions: Mapped[list[ApprovalDecision]] = relationship( diff --git a/packages/db/migrations/versions/0026_approval_repository_columns.py b/packages/db/migrations/versions/0026_approval_repository_columns.py new file mode 100644 index 00000000..984593c6 --- /dev/null +++ b/packages/db/migrations/versions/0026_approval_repository_columns.py @@ -0,0 +1,84 @@ +"""approval repository persistence: requested_actor + escalated columns + +Backs the DB-backed ``SqlAlchemyApprovalRepository`` (apps/api) with the two +``approval_request`` columns the domain :class:`forge_approval.models.ApprovalRequest` +carries but the F36 schema (0019) had no home for: + +* ``requested_actor`` — the requesting actor reference ("system" | ":"). + It is what the approval inbox shows and the ``approval.requested`` event carries + (distinct from the resolvable ``requested_by`` user id), so a faithful repository + must round-trip it verbatim. +* ``escalated`` — set when a reviewer escalates a gate; persisted on the parent row + so the single server-side authorizer re-enforces the admin-only rule on every + subsequent resolve attempt after a reload (not only in-process). + +Both are additive, ``NOT NULL`` with a server default, so existing rows read a +sane value ("system" / false) and nothing else in the schema is touched — the +in-memory repository (the unit-test default) is unaffected. + +Foundation note (mirrors 0019/0024/0025): ``forge_db``'s metadata is the source +of truth, so a fresh chain already provisions these columns from the model. To +stay idiomatic *and* own an explicit, reversible step this migration is +**idempotent**: ``upgrade`` adds only what is missing, ``downgrade`` drops only +what this revision introduced. Applies cleanly on SQLite (unit path) and the +pgvector Postgres test DB (:5433). + +Revision ID: 0026_approval_repository_columns +Revises: 0025_observability_audit_store +Create Date: 2026-07-05 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) + +# revision identifiers, used by Alembic. +revision: str = "0026_approval_repository_columns" +down_revision: str | None = "0025_observability_audit_store" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_TABLE = "approval_request" +# Column names owned by this revision, in add order (downgrade drops in reverse). +_COLUMN_NAMES: tuple[str, ...] = ("requested_actor", "escalated") + + +def _new_columns() -> list[sa.Column]: + """Fresh Column objects per call (a Column may be bound to one table only).""" + return [ + sa.Column( + "requested_actor", + sa.String(length=64), + server_default=sa.text("'system'"), + nullable=False, + ), + sa.Column( + "escalated", + sa.Boolean(), + server_default=sa.text("false"), + nullable=False, + ), + ] + + +def _existing_columns() -> set[str]: + return {c["name"] for c in sa.inspect(op.get_bind()).get_columns(_TABLE)} + + +def upgrade() -> None: + columns = _existing_columns() + for column in _new_columns(): + if column.name not in columns: + op.add_column(_TABLE, column) + + +def downgrade() -> None: + columns = _existing_columns() + for name in reversed(_COLUMN_NAMES): + if name in columns: + op.drop_column(_TABLE, name) diff --git a/packages/db/tests/test_migration.py b/packages/db/tests/test_migration.py index 65260aba..7fa5734a 100644 --- a/packages/db/tests/test_migration.py +++ b/packages/db/tests/test_migration.py @@ -998,6 +998,41 @@ def test_f39_audit_chain_migration_up_down_and_backfill(alembic_config: Config) engine.dispose() +# F36-persist approval-repository columns, owned by 0026_approval_repository_columns. +APPROVAL_REPO_COLUMNS = {"requested_actor", "escalated"} + + +def test_approval_repository_columns_migration_up_down(alembic_config: Config) -> None: + """0026 adds the ``requested_actor`` + ``escalated`` columns to + ``approval_request`` (backing the DB-backed ``SqlAlchemyApprovalRepository``) + and drops exactly them on downgrade, leaving the table intact. + + (forge_db's baseline is metadata-driven, so a fresh chain provisions the + columns at 0001; like 0019, the 0026 step is idempotent about that and owns a + clean, reversible down.)""" + url = alembic_config.get_main_option("sqlalchemy.url") + assert url is not None + engine = create_engine(url) + try: + command.upgrade(alembic_config, "head") + inspector = inspect(engine) + cols = {c["name"] for c in inspector.get_columns("approval_request")} + assert cols >= APPROVAL_REPO_COLUMNS, ( + f"missing 0026 columns: {sorted(APPROVAL_REPO_COLUMNS - cols)}" + ) + + # Downgrade one step: 0026 columns gone, approval_request still present. + command.downgrade(alembic_config, "0025_observability_audit_store") + inspector = inspect(engine) + assert "approval_request" in inspector.get_table_names() + cols_after = {c["name"] for c in inspector.get_columns("approval_request")} + assert not (APPROVAL_REPO_COLUMNS & cols_after), "downgrade left 0026 columns" + + command.downgrade(alembic_config, "base") + finally: + engine.dispose() + + # --------------------------------------------------------------------------- # # HARD-11 — live-Postgres migration round-trip, per-revision walk, and # # data-preservation. These run only against a real pgvector Postgres (the # From 683507661eceef76c6713f478b914b074e0b9773 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 12:55:28 +0200 Subject: [PATCH 04/12] feat(db/policy-override-grant-store): Postgres persistence Co-Authored-By: Claude Fable 5 --- .../forge_api/services/approval_service.py | 22 +- .../policy_override_grant_store_db.py | 207 ++++++++++++ apps/api/forge_api/settings.py | 8 + .../test_policy_override_grant_store_db.py | 312 ++++++++++++++++++ .../forge_approval/providers/__init__.py | 2 + .../providers/policy_override.py | 30 +- 6 files changed, 577 insertions(+), 4 deletions(-) create mode 100644 apps/api/forge_api/services/policy_override_grant_store_db.py create mode 100644 apps/api/tests/test_policy_override_grant_store_db.py diff --git a/apps/api/forge_api/services/approval_service.py b/apps/api/forge_api/services/approval_service.py index f8549ea6..464f9cbd 100644 --- a/apps/api/forge_api/services/approval_service.py +++ b/apps/api/forge_api/services/approval_service.py @@ -38,6 +38,7 @@ from forge_approval.providers import ( DeployGateProvider, DeployResolutionHook, + GrantStore, InMemoryGrantStore, PolicyOverrideGateProvider, PolicyOverrideResolutionHook, @@ -45,7 +46,7 @@ from forge_contracts import UserRole -def build_gate_registry(grants: InMemoryGrantStore) -> GateRegistry: +def build_gate_registry(grants: GrantStore) -> GateRegistry: """Register every available provider/hook (the F36 composition root).""" registry = GateRegistry() registry.register_provider(DeployGateProvider()) @@ -56,8 +57,23 @@ def build_gate_registry(grants: InMemoryGrantStore) -> GateRegistry: @lru_cache(maxsize=1) -def get_override_grant_store() -> InMemoryGrantStore: - """Process-wide single-use override-grant store (J5 consume contract).""" +def get_override_grant_store() -> GrantStore: + """Return the override-grant store selected by ``FORGE_OVERRIDE_GRANT_BACKEND``. + + ``memory`` (default) -> the hermetic :class:`InMemoryGrantStore` (unit-test + default, no Postgres); ``db`` -> the durable + :class:`~forge_api.services.policy_override_grant_store_db.DbGrantStore` bound + to the shared session factory. Both satisfy the same ``mint`` / ``consume`` / + ``all`` grant-store seam, so the swap is behaviour-preserving (single-active, + single-use, TTL-expiry). + """ + from forge_api.settings import get_settings + + if get_settings().override_grant_backend == "db": + from forge_api.db import get_session_factory + from forge_api.services.policy_override_grant_store_db import DbGrantStore + + return DbGrantStore(get_session_factory()) return InMemoryGrantStore() diff --git a/apps/api/forge_api/services/policy_override_grant_store_db.py b/apps/api/forge_api/services/policy_override_grant_store_db.py new file mode 100644 index 00000000..2cf3d020 --- /dev/null +++ b/apps/api/forge_api/services/policy_override_grant_store_db.py @@ -0,0 +1,207 @@ +"""Postgres-backed policy-override grant store (F36 J5 mint/consume seam). + +:class:`DbGrantStore` is a drop-in, durable alternative to +:class:`~forge_approval.providers.policy_override.InMemoryGrantStore` that +satisfies the **same** ``GrantStore`` seam (``mint`` / async ``consume`` / +``all``) the ``policy_override`` resolution hook + F06/F29 resume path depend on +— so the F36 composition root swaps it in behind +``FORGE_OVERRIDE_GRANT_BACKEND=db`` with no behavioural change. The default stays +``memory`` and the in-memory store remains the unit-test default. + +It lives in ``apps/api`` (not the frozen, DB-free ``forge_approval`` SDK), exactly +like the sibling :class:`~forge_api.services.approval_repository_db.SqlAlchemyApprovalRepository` +and :class:`~forge_api.observability.audit_db.DbAuditStore`: the SDK stays pure +domain and this adapter maps the domain +:class:`~forge_approval.models.PolicyOverrideGrant` onto the canonical +``policy_override_grant`` ORM row in ``forge_db`` (created by migration 0019, with +the partial-unique ``uq_active_override`` on active — unconsumed — grants). + +The single-active + single-use DB invariants are enforced *by the database*: + +* **single-active** — ``mint`` selects the active (unconsumed, unexpired) grant + ``FOR UPDATE`` and returns it verbatim when present (idempotent, mirroring the + in-memory store); otherwise it reaps any stale unconsumed-but-expired row (so + the partial unique index is free) and inserts a fresh grant. A concurrent + racing insert trips ``uq_active_override`` and is resolved by returning the + winner — never a duplicate active grant. +* **single-use** — ``consume`` is one atomic ``UPDATE ... SET consumed = true + WHERE active`` (the exact statement the worker resume task runs); its rowcount + decides the boolean, so double-consumption is impossible even across workers. +* **TTL expiry** — every active check carries ``expires_at > now``; an expired + grant denies on ``consume`` and does not block a fresh ``mint``. + +One storage-boundary detail (shared with every DB-backed repo here): the row is +workspace-scoped (``policy_override_grant`` is a ``WorkspaceScopedModel``) but the +domain grant carries no workspace — the tenant is derived from the ``agent_run`` +the grant is bound to, which is authoritative and always present in a coherent DB +deployment (the FK would otherwise reject the insert anyway). +""" + +from __future__ import annotations + +import builtins +import uuid +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from sqlalchemy import select, update +from sqlalchemy.exc import IntegrityError + +from forge_approval.models import PolicyOverrideGrant +from forge_db.models import AgentRun +from forge_db.models import PolicyOverrideGrant as PolicyOverrideGrantRow + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["DbGrantStore"] + + +class UnknownAgentRunError(LookupError): + """A grant references an ``agent_run`` that does not exist in the database.""" + + def __init__(self, agent_run_id: uuid.UUID) -> None: + super().__init__(f"unknown agent_run {agent_run_id}; cannot derive workspace") + self.agent_run_id = agent_run_id + + +def _aware(value: datetime | None) -> datetime | None: + """Normalise a stored timestamp to timezone-aware UTC (SQLite reads naive).""" + if value is None: + return None + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +class DbGrantStore: + """A Postgres-backed policy-override grant store (implements ``GrantStore``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + # ------------------------------------------------------------------ # + # Mapping # + # ------------------------------------------------------------------ # + + def _to_domain(self, row: PolicyOverrideGrantRow) -> PolicyOverrideGrant: + """Rebuild the domain grant from a persisted row.""" + return PolicyOverrideGrant( + id=row.id, + approval_request_id=row.approval_request_id, + agent_run_id=row.agent_run_id, + action_fingerprint=row.action_fingerprint, + granted_by=row.granted_by, + consumed=row.consumed, + expires_at=_aware(row.expires_at), # type: ignore[arg-type] + created_at=_aware(row.created_at), + ) + + # ------------------------------------------------------------------ # + # GrantStore seam # + # ------------------------------------------------------------------ # + + def mint(self, grant: PolicyOverrideGrant) -> PolicyOverrideGrant: + """Store a grant; at most one active per (agent_run_id, fingerprint). + + Returns the existing active grant unchanged when one is present + (idempotent, exactly like the in-memory store); otherwise inserts a + fresh grant, reaping any stale unconsumed-but-expired row first so the + partial-unique ``uq_active_override`` never blocks a legitimate re-mint. + """ + now = datetime.now(UTC) + with self._sf() as session: + # Lock every unconsumed row for this (run, fingerprint) so the + # check-reap-insert below is atomic against a concurrent mint. + unconsumed = session.scalars( + select(PolicyOverrideGrantRow) + .where( + PolicyOverrideGrantRow.agent_run_id == grant.agent_run_id, + PolicyOverrideGrantRow.action_fingerprint == grant.action_fingerprint, + PolicyOverrideGrantRow.consumed.is_(False), + ) + .with_for_update() + ).all() + active = next( + (r for r in unconsumed if _aware(r.expires_at) > now), # type: ignore[operator] + None, + ) + if active is not None: + return self._to_domain(active) + + # Any remaining unconsumed row is expired garbage — reap it (marking + # consumed) to free the WHERE consumed = false partial unique index. + for stale in unconsumed: + stale.consumed = True + + workspace_id = session.scalars( + select(AgentRun.workspace_id).where(AgentRun.id == grant.agent_run_id) + ).first() + if workspace_id is None: + raise UnknownAgentRunError(grant.agent_run_id) + + row = PolicyOverrideGrantRow( + id=grant.id, + workspace_id=workspace_id, + approval_request_id=grant.approval_request_id, + agent_run_id=grant.agent_run_id, + action_fingerprint=grant.action_fingerprint, + granted_by=grant.granted_by, + consumed=grant.consumed, + expires_at=grant.expires_at, + ) + if grant.created_at is not None: + row.created_at = grant.created_at + session.add(row) + try: + session.commit() + except IntegrityError: + # A concurrent mint won the race and inserted the active grant + # (uq_active_override); return the winner rather than duplicate. + session.rollback() + winner = session.scalars( + select(PolicyOverrideGrantRow) + .where( + PolicyOverrideGrantRow.agent_run_id == grant.agent_run_id, + PolicyOverrideGrantRow.action_fingerprint + == grant.action_fingerprint, + PolicyOverrideGrantRow.consumed.is_(False), + ) + .with_for_update() + ).first() + if winner is not None: + return self._to_domain(winner) + raise + return self._to_domain(row) + + async def consume( + self, *, agent_run_id: uuid.UUID, action_fingerprint: str + ) -> bool: + """Atomically consume the active grant for this exact action, if any. + + Returns ``True`` only when a matching unconsumed, unexpired grant was + flipped by *this* single ``UPDATE`` — never granting future scope. + """ + now = datetime.now(UTC) + with self._sf() as session: + result = session.execute( + update(PolicyOverrideGrantRow) + .where( + PolicyOverrideGrantRow.agent_run_id == agent_run_id, + PolicyOverrideGrantRow.action_fingerprint == action_fingerprint, + PolicyOverrideGrantRow.consumed.is_(False), + PolicyOverrideGrantRow.expires_at > now, + ) + .values(consumed=True) + ) + session.commit() + return (result.rowcount or 0) > 0 + + def all(self) -> builtins.list[PolicyOverrideGrant]: + """Every stored grant, in stable insertion order.""" + with self._sf() as session: + rows = session.scalars( + select(PolicyOverrideGrantRow).order_by( + PolicyOverrideGrantRow.created_at.asc(), + PolicyOverrideGrantRow.id.asc(), + ) + ).all() + return [self._to_domain(r) for r in rows] diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index dc59a5f4..a95546e2 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -126,6 +126,14 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # are durably persisted. Read via ``FORGE_APPROVAL_BACKEND``. approval_backend: str = "memory" + # F36 policy-override grant-store backend selection (J5). ``memory`` (default) + # keeps the hermetic, process-memory ``InMemoryGrantStore`` (unit-test default, + # no Postgres); ``db`` wires the Postgres-backed ``DbGrantStore`` behind the + # same ``mint`` / ``consume`` / ``all`` grant-store seam so single-use override + # grants survive a restart and the single-active + atomic-consume invariants + # are enforced by the database. Read via ``FORGE_OVERRIDE_GRANT_BACKEND``. + override_grant_backend: str = "memory" + # Filesystem root for the spec engine's SDD artifacts (manifests, plans). spec_root: str = "specs" diff --git a/apps/api/tests/test_policy_override_grant_store_db.py b/apps/api/tests/test_policy_override_grant_store_db.py new file mode 100644 index 00000000..d1adcd0a --- /dev/null +++ b/apps/api/tests/test_policy_override_grant_store_db.py @@ -0,0 +1,312 @@ +"""Postgres integration tests for :class:`DbGrantStore` (F36 J5). + +Exercises the DB-backed policy-override grant store against a real pgvector +Postgres via the shared ``pg_engine`` fixture (root ``conftest.py``): the +``mint`` / async ``consume`` / ``all`` grant-store seam end-to-end — a full +``PolicyOverrideGrant`` round-trip, the single-active invariant (idempotent +mint + the ``uq_active_override`` partial-unique storage boundary), atomic +single-use ``consume``, TTL expiry (expired denies + does not block a re-mint), +fingerprint/agent-run mismatch denials, workspace derivation from the bound +``agent_run``, durability across store instances, and structural conformance to +the same seam the in-memory store implements. + +Skips cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. Each behaviour mirrors +the in-memory contract, so both backends are proven to satisfy the same seam +identically. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.services.policy_override_grant_store_db import ( + DbGrantStore, + UnknownAgentRunError, +) +from forge_approval.models import PolicyOverrideGrant +from forge_approval.providers.policy_override import ( + GrantStore, + InMemoryGrantStore, + PolicyOverrideGate, + action_fingerprint, +) +from forge_db.base import Base +from forge_db.models import AgentRun, ApprovalRequest, User, Workspace +from forge_db.models.enums import ApprovalGate + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + +FINGERPRINT = action_fingerprint( + {"tool": "shell", "action": "run", "arguments": {"cmd": "rm -rf /tmp/x"}} +) +OTHER_FINGERPRINT = action_fingerprint( + {"tool": "shell", "action": "run", "arguments": {"cmd": "ls"}} +) + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def seed(factory: sessionmaker[Session]) -> dict[str, uuid.UUID]: + """A workspace + admin user + agent_run + approval_request (the grant FKs).""" + ws = uuid.uuid4() + admin = uuid.uuid4() + run = uuid.uuid4() + request = uuid.uuid4() + with factory() as session: + session.add(Workspace(id=ws, name="Acme", slug=f"acme-{uuid.uuid4().hex[:8]}")) + session.flush() + session.add( + User(id=admin, workspace_id=ws, email=f"a-{admin.hex[:6]}@acme.dev", name="Admin") + ) + session.add(AgentRun(id=run, workspace_id=ws)) + session.add( + ApprovalRequest(id=request, workspace_id=ws, gate=ApprovalGate.POLICY_OVERRIDE) + ) + session.commit() + return {"ws": ws, "admin": admin, "run": run, "request": request} + + +@pytest.fixture +def store(factory: sessionmaker[Session]) -> DbGrantStore: + return DbGrantStore(factory) + + +def _grant( + seed: dict[str, uuid.UUID], + *, + fingerprint: str = FINGERPRINT, + expires_in: timedelta = timedelta(minutes=15), + grant_id: uuid.UUID | None = None, +) -> PolicyOverrideGrant: + return PolicyOverrideGrant( + id=grant_id or uuid.uuid4(), + approval_request_id=seed["request"], + agent_run_id=seed["run"], + action_fingerprint=fingerprint, + granted_by=seed["admin"], + expires_at=datetime.now(UTC) + expires_in, + ) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_store_satisfies_grant_store_and_gate_protocols(store: DbGrantStore) -> None: + assert isinstance(store, GrantStore) + assert isinstance(store, PolicyOverrideGate) # the consume-only resume contract + + +# --------------------------------------------------------------------------- # +# mint + round-trip # +# --------------------------------------------------------------------------- # + + +def test_mint_round_trips_every_field( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + grant = _grant(seed) + returned = store.mint(grant) + assert returned.id == grant.id + assert returned.created_at is not None # server-default timestamp populated + + stored = store.all() + assert len(stored) == 1 + row = stored[0] + assert row.id == grant.id + assert row.approval_request_id == seed["request"] + assert row.agent_run_id == seed["run"] + assert row.action_fingerprint == FINGERPRINT + assert row.granted_by == seed["admin"] + assert row.consumed is False + assert row.expires_at is not None and row.expires_at.tzinfo is not None + assert row.expires_at == grant.expires_at + + +def test_mint_derives_workspace_from_agent_run( + store: DbGrantStore, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + from sqlalchemy import select + + from forge_db.models import PolicyOverrideGrant as Row + + store.mint(_grant(seed)) + with factory() as session: + row = session.scalars(select(Row)).one() + assert row.workspace_id == seed["ws"] + + +def test_mint_unknown_agent_run_raises( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + ghost = seed | {"run": uuid.uuid4()} + with pytest.raises(UnknownAgentRunError): + store.mint(_grant(ghost)) + + +# --------------------------------------------------------------------------- # +# single-active invariant # +# --------------------------------------------------------------------------- # + + +def test_mint_is_idempotent_while_active( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + first = store.mint(_grant(seed)) + second = store.mint(_grant(seed)) # a *different* grant object, same (run, fp) + assert second.id == first.id # existing active grant returned, not duplicated + assert len(store.all()) == 1 + + +# --------------------------------------------------------------------------- # +# consume: single-use + atomic # +# --------------------------------------------------------------------------- # + + +async def test_consume_is_single_use( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + store.mint(_grant(seed)) + assert await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + # ... and never again for the same grant. + assert not await store.consume( + agent_run_id=seed["run"], action_fingerprint=FINGERPRINT + ) + + +async def test_consume_without_grant_denies( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + assert not await store.consume( + agent_run_id=seed["run"], action_fingerprint=FINGERPRINT + ) + + +async def test_consume_fingerprint_mismatch_denies( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + store.mint(_grant(seed)) + assert not await store.consume( + agent_run_id=seed["run"], action_fingerprint=OTHER_FINGERPRINT + ) + # The original grant is untouched by the mismatch and still consumable once. + assert await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + + +async def test_consume_agent_run_mismatch_denies( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + store.mint(_grant(seed)) + assert not await store.consume( + agent_run_id=uuid.uuid4(), action_fingerprint=FINGERPRINT + ) + + +# --------------------------------------------------------------------------- # +# TTL expiry # +# --------------------------------------------------------------------------- # + + +async def test_expired_grant_denies_consume( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + store.mint(_grant(seed, expires_in=timedelta(minutes=-1))) + assert not await store.consume( + agent_run_id=seed["run"], action_fingerprint=FINGERPRINT + ) + + +async def test_remint_after_expiry_yields_a_fresh_usable_grant( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + """An expired-but-unconsumed grant must not block a re-mint (index reap).""" + expired = store.mint(_grant(seed, expires_in=timedelta(minutes=-1))) + fresh = store.mint(_grant(seed, expires_in=timedelta(minutes=15))) + assert fresh.id != expired.id # a genuinely new active grant + # The fresh grant is consumable; the stale one was reaped out of the way. + assert await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + + +# --------------------------------------------------------------------------- # +# single-active across consumption + storage boundary # +# --------------------------------------------------------------------------- # + + +async def test_remint_after_consume_creates_new_active_grant( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + first = store.mint(_grant(seed)) + assert await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + second = store.mint(_grant(seed)) # once consumed, a fresh active grant may mint + assert second.id != first.id + assert len(store.all()) == 2 # both rows persist (one consumed, one active) + assert await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + + +def test_all_orders_by_insertion( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + # Two distinct fingerprints so both stay active (single-active is per-fp). + g1 = store.mint(_grant(seed, fingerprint=FINGERPRINT)) + g2 = store.mint(_grant(seed, fingerprint=OTHER_FINGERPRINT)) + assert [g.id for g in store.all()] == [g1.id, g2.id] + + +# --------------------------------------------------------------------------- # +# durability # +# --------------------------------------------------------------------------- # + + +async def test_persists_across_store_instances( + factory: sessionmaker[Session], seed: dict[str, uuid.UUID] +) -> None: + first = DbGrantStore(factory) + first.mint(_grant(seed)) + + second = DbGrantStore(factory) + assert len(second.all()) == 1 + # A grant minted through one instance is consumable through another. + assert await second.consume( + agent_run_id=seed["run"], action_fingerprint=FINGERPRINT + ) + + +# --------------------------------------------------------------------------- # +# Parity with the in-memory store (same seam, identical behaviour) # +# --------------------------------------------------------------------------- # + + +async def test_matches_in_memory_store_behaviour( + store: DbGrantStore, seed: dict[str, uuid.UUID] +) -> None: + mem = InMemoryGrantStore() + + db_first = store.mint(_grant(seed, grant_id=uuid.uuid4())) + mem_first = mem.mint(_grant(seed, grant_id=db_first.id)) + + # Idempotent-while-active: both return the existing grant, not a duplicate. + assert store.mint(_grant(seed)).id == db_first.id + assert mem.mint(_grant(seed, grant_id=mem_first.id)).id == mem_first.id + + # Single-use: True exactly once on both backends, then False. + assert ( + await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + ) == (await mem.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT)) + assert ( + await store.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT) + ) == (await mem.consume(agent_run_id=seed["run"], action_fingerprint=FINGERPRINT)) diff --git a/packages/approval-sdk/forge_approval/providers/__init__.py b/packages/approval-sdk/forge_approval/providers/__init__.py index 72a10622..2a7f80c5 100644 --- a/packages/approval-sdk/forge_approval/providers/__init__.py +++ b/packages/approval-sdk/forge_approval/providers/__init__.py @@ -8,6 +8,7 @@ from forge_approval.providers.deploy import DeployGateProvider, DeployResolutionHook from forge_approval.providers.policy_override import ( + GrantStore, InMemoryGrantStore, PolicyOverrideGate, PolicyOverrideGateProvider, @@ -18,6 +19,7 @@ __all__ = [ "DeployGateProvider", "DeployResolutionHook", + "GrantStore", "InMemoryGrantStore", "PolicyOverrideGate", "PolicyOverrideGateProvider", diff --git a/packages/approval-sdk/forge_approval/providers/policy_override.py b/packages/approval-sdk/forge_approval/providers/policy_override.py index a46e3c26..7d1f67c1 100644 --- a/packages/approval-sdk/forge_approval/providers/policy_override.py +++ b/packages/approval-sdk/forge_approval/providers/policy_override.py @@ -53,6 +53,33 @@ async def consume(self, *, agent_run_id: uuid.UUID, action_fingerprint: str) -> ... +@runtime_checkable +class GrantStore(Protocol): + """The full ``mint`` / ``consume`` / ``all`` grant-store seam. + + Superset of :class:`PolicyOverrideGate` (the consume-only resume contract): + the resolution hook mints grants and the composition root/inspection reads + them all. :class:`InMemoryGrantStore` is the hermetic default; a Postgres + adapter satisfies this *same* Protocol, so the store is swappable behind an + env flag with no behaviour change. + """ + + def mint(self, grant: PolicyOverrideGrant) -> PolicyOverrideGrant: + """Store a grant; at most one active per (agent_run_id, fingerprint). + + Idempotent while an active grant exists: returns the existing one + rather than a duplicate.""" + ... + + async def consume(self, *, agent_run_id: uuid.UUID, action_fingerprint: str) -> bool: + """Atomically check-and-consume a non-expired grant (single-use).""" + ... + + def all(self) -> list[PolicyOverrideGrant]: + """Every stored grant (active, consumed, or expired).""" + ... + + class InMemoryGrantStore: """Grant store honouring the single-active + single-use DB invariants.""" @@ -159,7 +186,7 @@ class PolicyOverrideResolutionHook: gate_type: ClassVar[GateType] = GateType.POLICY_OVERRIDE def __init__( - self, grants: InMemoryGrantStore, *, ttl: timedelta = DEFAULT_GRANT_TTL + self, grants: GrantStore, *, ttl: timedelta = DEFAULT_GRANT_TTL ) -> None: self._grants = grants self._ttl = ttl @@ -216,6 +243,7 @@ async def on_resolved( __all__ = [ "DEFAULT_GRANT_TTL", "POLICY_OVERRIDE_GRANTED_SIGNAL", + "GrantStore", "InMemoryGrantStore", "PolicyOverrideGate", "PolicyOverrideGateProvider", From 237285f926e1f36a6a0b2d61254393b7f0be0537 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 13:47:22 +0200 Subject: [PATCH 05/12] feat(db/api-key-backend): Postgres persistence Co-Authored-By: Claude Fable 5 --- apps/api/forge_api/auth/apikeys_db.py | 216 ++++++++++++++ apps/api/forge_api/auth/service.py | 27 +- apps/api/forge_api/settings.py | 8 + apps/api/tests/test_apikeys_db.py | 415 ++++++++++++++++++++++++++ 4 files changed, 664 insertions(+), 2 deletions(-) create mode 100644 apps/api/forge_api/auth/apikeys_db.py create mode 100644 apps/api/tests/test_apikeys_db.py diff --git a/apps/api/forge_api/auth/apikeys_db.py b/apps/api/forge_api/auth/apikeys_db.py new file mode 100644 index 00000000..698587c4 --- /dev/null +++ b/apps/api/forge_api/auth/apikeys_db.py @@ -0,0 +1,216 @@ +"""Postgres-backed platform API-key backend (Phase-2 persistence). + +:class:`DbAPIKeyBackend` is a drop-in, durable alternative to +:class:`~forge_api.auth.apikeys.InMemoryAPIKeyBackend` that satisfies the **same** +:class:`~forge_api.auth.apikeys.APIKeyBackend` seam (``add`` / ``by_prefix`` / +``list`` / ``get``) the :class:`~forge_api.auth.apikeys.APIKeyStore` mints, verifies, +lists, and revokes through. The composition root swaps it in behind +``FORGE_APIKEY_BACKEND=db``; the default stays ``memory`` and the in-memory store +remains the unit-test default, so no existing behaviour changes. + +It maps the domain :class:`~forge_api.auth.apikeys.APIKeyRecord` onto the canonical +``platform_api_key`` ORM row (``PlatformAPIKey``, created by migration 0020, F37) — +so **no new migration** is required. Two storage-boundary details are load-bearing: + +* **Enum taxonomy.** The record carries an + :class:`~forge_contracts.enums.APIKeyKind` (BYOK-flavoured; the store only ever + mints ``SYSTEM`` for platform auth), while the frozen ``platform_api_key.kind`` + column is a :class:`~forge_contracts.auth.PlatformKeyKind`. The two are bridged + by :data:`_KIND_TO_PLATFORM` / :data:`_PLATFORM_TO_KIND`, a documented mapping + that round-trips the platform-auth kinds (``SYSTEM`` ⇄ ``service``, + ``MODEL_PROVIDER`` ⇄ ``personal``) verbatim and never emits ``agent_runner`` on + write (so the ``agent_runner ⇒ expires_at`` CHECK is never tripped). The two + BYOK-only kinds that never reach the platform-key store fold onto those two + slots and read back as their canonical twin (documented, unreachable in practice). + +* **Mutation through returned records.** ``APIKeyStore.revoke`` / + ``revoke_for_user`` flip ``is_active`` and ``verify`` stamps ``last_used_at`` by + mutating the record object the backend returns — the in-memory store persists + that only because it hands back live references. To preserve that behaviour + exactly, this backend returns :class:`_LiveAPIKeyRecord` instances that + write-through those two fields (``is_active`` → ``revoked_at``; ``last_used_at``) + to the row on assignment. Revocation and last-used tracking therefore work + identically on both backends. +""" + +from __future__ import annotations + +import builtins +import uuid +from collections.abc import Callable +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from pydantic import PrivateAttr +from sqlalchemy import select, update + +from forge_api.auth.apikeys import APIKeyRecord +from forge_contracts.auth import PlatformKeyKind +from forge_contracts.enums import APIKeyKind, UserRole +from forge_db.models import PlatformAPIKey +from forge_db.models.enums import UserRole as DbUserRole + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["DbAPIKeyBackend"] + +#: Forward map: the record's :class:`APIKeyKind` → the column's +#: :class:`PlatformKeyKind`. ``agent_runner`` is deliberately never a target so the +#: ``agent_runner ⇒ expires_at`` CHECK is never at risk; the platform-auth kinds +#: (``SYSTEM``/``MODEL_PROVIDER``) map onto distinct slots and round-trip verbatim. +_KIND_TO_PLATFORM: dict[APIKeyKind, PlatformKeyKind] = { + APIKeyKind.SYSTEM: PlatformKeyKind.SERVICE, + APIKeyKind.MODEL_PROVIDER: PlatformKeyKind.PERSONAL, + APIKeyKind.INTEGRATION_TOKEN: PlatformKeyKind.SERVICE, + APIKeyKind.MCP_TOKEN: PlatformKeyKind.PERSONAL, +} + +#: Reverse map for reads. Total over :class:`PlatformKeyKind`; ``agent_runner`` is +#: covered defensively (this backend never writes it) and reads back as ``SYSTEM``. +_PLATFORM_TO_KIND: dict[PlatformKeyKind, APIKeyKind] = { + PlatformKeyKind.SERVICE: APIKeyKind.SYSTEM, + PlatformKeyKind.PERSONAL: APIKeyKind.MODEL_PROVIDER, + PlatformKeyKind.AGENT_RUNNER: APIKeyKind.SYSTEM, +} + +#: Record fields whose in-place mutation the store relies on the backend to +#: persist (the in-memory store gets this free via shared references). +_WRITE_THROUGH_FIELDS = frozenset({"is_active", "last_used_at"}) + + +def _aware(value: datetime | None) -> datetime | None: + """Normalise a stored timestamp to timezone-aware UTC (defensive).""" + if value is None: + return None + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +class _LiveAPIKeyRecord(APIKeyRecord): + """An :class:`APIKeyRecord` that write-throughs revoke/last-used mutations. + + ``APIKeyStore`` mutates ``is_active`` / ``last_used_at`` on the record objects + the backend returns; the in-memory store persists those only because it hands + back the very objects it stores. This subclass reproduces that behaviour for + the DB backend by forwarding those two assignments to a persistence hook, so a + ``revoke`` / ``revoke_for_user`` / ``verify`` behaves identically on both. + """ + + _persist: Callable[[uuid.UUID, str, Any], None] | None = PrivateAttr(default=None) + + def __setattr__(self, name: str, value: Any) -> None: + super().__setattr__(name, value) + private = getattr(self, "__pydantic_private__", None) + hook = private.get("_persist") if private else None + if hook is not None and name in _WRITE_THROUGH_FIELDS: + hook(self.id, name, value) + + +class DbAPIKeyBackend: + """A Postgres-backed API-key backend (implements ``APIKeyBackend``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + # ------------------------------------------------------------------ # + # Mapping # + # ------------------------------------------------------------------ # + + def _to_row(self, record: APIKeyRecord) -> PlatformAPIKey: + """Build the ORM row for a domain record (kind/role/active translated).""" + return PlatformAPIKey( + id=record.id, + workspace_id=record.workspace_id, + name=record.name, + # Non-secret unique lookup id the column requires (the record has no + # ``key_id`` of its own); derived from the unique record id, stable + # across re-adds and never surfaced back into the domain. + key_id=record.id.hex[:16], + key_hash=record.token_hash, + key_prefix=record.key_prefix, + kind=_KIND_TO_PLATFORM[record.kind], + role=DbUserRole(record.role.value), + created_by=record.user_id, + created_at=record.created_at, + last_used_at=record.last_used_at, + expires_at=record.expires_at, + revoked_at=None if record.is_active else datetime.now(UTC), + ) + + def _to_record(self, row: PlatformAPIKey) -> _LiveAPIKeyRecord: + """Rebuild a live (write-through) domain record from a persisted row.""" + record = _LiveAPIKeyRecord( + id=row.id, + workspace_id=row.workspace_id, + name=row.name, + kind=_PLATFORM_TO_KIND[row.kind], + role=UserRole(row.role.value), + key_prefix=row.key_prefix, + token_hash=row.key_hash, + user_id=row.created_by, + created_at=_aware(row.created_at), # type: ignore[arg-type] + last_used_at=_aware(row.last_used_at), + expires_at=_aware(row.expires_at), + is_active=row.revoked_at is None, + ) + record._persist = self._persist_field + return record + + # ------------------------------------------------------------------ # + # Write-through persistence for in-place record mutation # + # ------------------------------------------------------------------ # + + def _persist_field(self, key_id: uuid.UUID, name: str, value: Any) -> None: + """Persist a mutation of ``is_active`` / ``last_used_at`` on one row.""" + if name == "last_used_at": + values: dict[str, Any] = {"last_used_at": value} + else: # is_active → revoked_at (kept for audit; None re-activates) + values = {"revoked_at": None if value else datetime.now(UTC)} + with self._sf() as session: + session.execute( + update(PlatformAPIKey).where(PlatformAPIKey.id == key_id).values(**values) + ) + session.commit() + + # ------------------------------------------------------------------ # + # APIKeyBackend seam # + # ------------------------------------------------------------------ # + + def add(self, record: APIKeyRecord) -> None: + """Persist a record; overwrites on a repeated id (mirrors the dict store).""" + with self._sf() as session: + session.merge(self._to_row(record)) + session.commit() + + def by_prefix(self, prefix: str) -> builtins.list[APIKeyRecord]: + """Every record whose display prefix matches, oldest first (stable).""" + with self._sf() as session: + rows = session.scalars( + select(PlatformAPIKey) + .where(PlatformAPIKey.key_prefix == prefix) + .order_by(PlatformAPIKey.created_at.asc(), PlatformAPIKey.id.asc()) + ).all() + return [self._to_record(r) for r in rows] + + def list(self, workspace_id: uuid.UUID) -> builtins.list[APIKeyRecord]: + """Every record in a workspace, oldest first (stable ordering).""" + with self._sf() as session: + rows = session.scalars( + select(PlatformAPIKey) + .where(PlatformAPIKey.workspace_id == workspace_id) + .order_by(PlatformAPIKey.created_at.asc(), PlatformAPIKey.id.asc()) + ).all() + return [self._to_record(r) for r in rows] + + def get( + self, workspace_id: uuid.UUID, key_id: uuid.UUID + ) -> APIKeyRecord | None: + """The record with ``key_id`` in ``workspace_id``, else ``None``.""" + with self._sf() as session: + row = session.scalars( + select(PlatformAPIKey).where( + PlatformAPIKey.id == key_id, + PlatformAPIKey.workspace_id == workspace_id, + ) + ).first() + return self._to_record(row) if row is not None else None diff --git a/apps/api/forge_api/auth/service.py b/apps/api/forge_api/auth/service.py index 954e93d2..6cce02b5 100644 --- a/apps/api/forge_api/auth/service.py +++ b/apps/api/forge_api/auth/service.py @@ -31,7 +31,7 @@ from fastapi import Depends, Header, HTTPException, status -from forge_api.auth.apikeys import APIKeyInfo, APIKeyStore +from forge_api.auth.apikeys import APIKeyBackend, APIKeyInfo, APIKeyStore from forge_api.auth.crypto import EnvelopeCipher, default_cipher from forge_api.auth.keyring import KeyRing from forge_api.auth.models import OAuthChallenge, OAuthResult @@ -168,6 +168,26 @@ def _resolve_master_key(secret_key: bytes | None) -> bytes: ) +def _build_apikey_backend() -> APIKeyBackend | None: + """Return the API-key backend selected by ``FORGE_APIKEY_BACKEND``. + + ``memory`` (default) → ``None``, so :class:`APIKeyStore` falls back to the + hermetic :class:`~forge_api.auth.apikeys.InMemoryAPIKeyBackend` (unit-test + default, no Postgres); ``db`` → the durable + :class:`~forge_api.auth.apikeys_db.DbAPIKeyBackend` bound to the shared + session factory. Both satisfy the same ``APIKeyBackend`` seam, so the swap is + behaviour-preserving (mint / verify / list / revoke). + """ + from forge_api.settings import get_settings + + if get_settings().apikey_backend == "db": + from forge_api.auth.apikeys_db import DbAPIKeyBackend + from forge_api.db import get_session_factory + + return DbAPIKeyBackend(get_session_factory()) + return None + + def _keyring_for_master(master: bytes) -> KeyRing: """Build a :class:`KeyRing` whose current KEK is the resolved ``master`` key. @@ -212,7 +232,10 @@ def __init__( audit_sink: AuditSink | None = None, ) -> None: master = _resolve_master_key(secret_key) - self.api_keys = api_keys or APIKeyStore(secret_key=_subkey(master, b"forge-apikey")) + self.api_keys = api_keys or APIKeyStore( + secret_key=_subkey(master, b"forge-apikey"), + backend=_build_apikey_backend(), + ) self.vault = vault or _build_vault(master) # Constructs without network access; the IdP is only contacted when an # authorization-code exchange is actually requested. diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index a95546e2..b337b158 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -126,6 +126,14 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # are durably persisted. Read via ``FORGE_APPROVAL_BACKEND``. approval_backend: str = "memory" + # Platform API-key backend selection. ``memory`` (default) keeps the hermetic, + # process-memory ``InMemoryAPIKeyBackend`` (unit-test default, no Postgres); + # ``db`` wires the Postgres-backed ``DbAPIKeyBackend`` behind the same + # ``APIKeyBackend`` seam (``add`` / ``by_prefix`` / ``list`` / ``get``) onto the + # ``platform_api_key`` table so minted keys, revocations, and last-used stamps + # survive a restart. Read via ``FORGE_APIKEY_BACKEND``. + apikey_backend: str = "memory" + # F36 policy-override grant-store backend selection (J5). ``memory`` (default) # keeps the hermetic, process-memory ``InMemoryGrantStore`` (unit-test default, # no Postgres); ``db`` wires the Postgres-backed ``DbGrantStore`` behind the diff --git a/apps/api/tests/test_apikeys_db.py b/apps/api/tests/test_apikeys_db.py new file mode 100644 index 00000000..c450c648 --- /dev/null +++ b/apps/api/tests/test_apikeys_db.py @@ -0,0 +1,415 @@ +"""Postgres integration tests for :class:`DbAPIKeyBackend`. + +Exercises the DB-backed platform API-key backend against a real pgvector Postgres +via the shared ``pg_engine`` fixture (root ``conftest.py``): the ``add`` / +``by_prefix`` / ``list`` / ``get`` seam end-to-end — full :class:`APIKeyRecord` +round-trip, workspace-scoped filtering + ordering, prefix indexing, the +``platform_api_key`` referential-integrity boundary, overwrite-on-re-add, and the +enum-taxonomy bridge — plus the full :class:`APIKeyStore` behaviours that flow +through the backend (mint → verify last-used stamp, revoke, revoke-for-user, +list) proving byte-for-byte parity with the in-memory store. + +Skips cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.auth.apikeys import ( + APIKeyBackend, + APIKeyRecord, + APIKeyStore, + InMemoryAPIKeyBackend, + _token_prefix, + generate_api_token, +) +from forge_api.auth.apikeys_db import DbAPIKeyBackend +from forge_contracts.auth import PlatformKeyKind +from forge_contracts.enums import APIKeyKind, UserRole +from forge_db.base import Base +from forge_db.models import PlatformAPIKey, User, Workspace + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + +SECRET = b"unit-test-apikey-subkey-0123456789" + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def ws(factory: sessionmaker[Session]) -> uuid.UUID: + """A persisted workspace (the ``platform_api_key.workspace_id`` FK target).""" + workspace_id = uuid.uuid4() + with factory() as session: + session.add( + Workspace(id=workspace_id, name="Acme", slug=f"acme-{uuid.uuid4().hex[:8]}") + ) + session.commit() + return workspace_id + + +@pytest.fixture +def other_ws(factory: sessionmaker[Session]) -> uuid.UUID: + workspace_id = uuid.uuid4() + with factory() as session: + session.add( + Workspace(id=workspace_id, name="Beta", slug=f"beta-{uuid.uuid4().hex[:8]}") + ) + session.commit() + return workspace_id + + +@pytest.fixture +def backend(factory: sessionmaker[Session]) -> DbAPIKeyBackend: + return DbAPIKeyBackend(factory) + + +def _record( + ws: uuid.UUID, + *, + kind: APIKeyKind = APIKeyKind.SYSTEM, + role: UserRole = UserRole.MEMBER, + token: str | None = None, + user_id: uuid.UUID | None = None, + created_at: datetime | None = None, + last_used_at: datetime | None = None, + expires_at: datetime | None = None, + is_active: bool = True, + record_id: uuid.UUID | None = None, +) -> APIKeyRecord: + tok = token or generate_api_token(kind) + return APIKeyRecord( + id=record_id or uuid.uuid4(), + workspace_id=ws, + name="ci", + kind=kind, + role=role, + key_prefix=_token_prefix(tok), + token_hash=f"hash-{uuid.uuid4().hex}", + user_id=user_id, + created_at=created_at or datetime.now(UTC), + last_used_at=last_used_at, + expires_at=expires_at, + is_active=is_active, + ) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_backend_satisfies_apikey_backend_protocol(backend: DbAPIKeyBackend) -> None: + assert isinstance(backend, APIKeyBackend) + + +# --------------------------------------------------------------------------- # +# add + round-trip # +# --------------------------------------------------------------------------- # + + +def test_add_round_trips_every_field(backend: DbAPIKeyBackend, ws: uuid.UUID) -> None: + created = datetime.now(UTC) - timedelta(minutes=5) + used = datetime.now(UTC) - timedelta(minutes=1) + expires = datetime.now(UTC) + timedelta(hours=1) + record = _record( + ws, + role=UserRole.ADMIN, + created_at=created, + last_used_at=used, + expires_at=expires, + ) + backend.add(record) + + got = backend.get(ws, record.id) + assert got is not None + assert got.id == record.id + assert got.workspace_id == ws + assert got.name == record.name + assert got.kind == APIKeyKind.SYSTEM # SYSTEM ⇄ service round-trips verbatim + assert got.role == UserRole.ADMIN + assert got.key_prefix == record.key_prefix + assert got.token_hash == record.token_hash + assert got.user_id is None + assert got.is_active is True + assert got.created_at == created + assert got.last_used_at == used + assert got.expires_at == expires + for value in (got.created_at, got.last_used_at, got.expires_at): + assert value.tzinfo is not None + + +def test_kind_bridge_persists_platform_kind( + backend: DbAPIKeyBackend, ws: uuid.UUID, factory: sessionmaker[Session] +) -> None: + backend.add(_record(ws, kind=APIKeyKind.SYSTEM)) + with factory() as session: + row = session.scalars(select(PlatformAPIKey)).one() + # Stored under the frozen column's taxonomy... + assert row.kind is PlatformKeyKind.SERVICE + # ...and read back as the record's own kind. + assert backend.list(ws)[0].kind == APIKeyKind.SYSTEM + + +def test_add_persists_user_id_as_created_by( + backend: DbAPIKeyBackend, ws: uuid.UUID, factory: sessionmaker[Session] +) -> None: + user_id = uuid.uuid4() + with factory() as session: + session.add( + User(id=user_id, workspace_id=ws, email=f"u-{user_id.hex[:6]}@acme.dev", name="U") + ) + session.commit() + backend.add(_record(ws, user_id=user_id)) + got = backend.list(ws)[0] + assert got.user_id == user_id + + +def test_add_overwrites_on_repeated_id( + backend: DbAPIKeyBackend, ws: uuid.UUID +) -> None: + """Re-adding the same id overwrites (mirrors the dict store's ``add``).""" + rid = uuid.uuid4() + backend.add(_record(ws, role=UserRole.MEMBER, record_id=rid)) + backend.add(_record(ws, role=UserRole.ADMIN, record_id=rid)) + keys = backend.list(ws) + assert len(keys) == 1 + assert keys[0].role == UserRole.ADMIN + + +# --------------------------------------------------------------------------- # +# referential-integrity boundary # +# --------------------------------------------------------------------------- # + + +def test_add_unknown_workspace_rejected(backend: DbAPIKeyBackend) -> None: + with pytest.raises(IntegrityError): + backend.add(_record(uuid.uuid4())) # workspace FK has no target row + + +# --------------------------------------------------------------------------- # +# by_prefix # +# --------------------------------------------------------------------------- # + + +def test_by_prefix_returns_only_matching( + backend: DbAPIKeyBackend, ws: uuid.UUID +) -> None: + tok_a = generate_api_token(APIKeyKind.SYSTEM) # prefix "forge_sy" + tok_b = generate_api_token(APIKeyKind.MCP_TOKEN) # distinct prefix "forge_mc" + a = _record(ws, token=tok_a) + backend.add(a) + backend.add(_record(ws, kind=APIKeyKind.MCP_TOKEN, token=tok_b)) + matches = backend.by_prefix(_token_prefix(tok_a)) + assert [m.id for m in matches] == [a.id] + assert matches[0].key_prefix == _token_prefix(tok_a) + + +def test_by_prefix_returns_all_sharing_a_prefix( + backend: DbAPIKeyBackend, ws: uuid.UUID +) -> None: + # Every SYSTEM token shares the display prefix "forge_sy" — the exact case the + # store's constant-time verify fans out over. + records = [_record(ws) for _ in range(3)] + prefix = records[0].key_prefix + assert all(r.key_prefix == prefix for r in records) + for rec in records: + backend.add(rec) + matches = backend.by_prefix(prefix) + assert {m.id for m in matches} == {r.id for r in records} + + +def test_by_prefix_unknown_is_empty(backend: DbAPIKeyBackend, ws: uuid.UUID) -> None: + backend.add(_record(ws)) + assert backend.by_prefix("forge_zz") == [] + + +# --------------------------------------------------------------------------- # +# list: workspace scoping + ordering # +# --------------------------------------------------------------------------- # + + +def test_list_is_workspace_scoped( + backend: DbAPIKeyBackend, ws: uuid.UUID, other_ws: uuid.UUID +) -> None: + backend.add(_record(ws)) + backend.add(_record(ws)) + backend.add(_record(other_ws)) + assert len(backend.list(ws)) == 2 + assert len(backend.list(other_ws)) == 1 + assert backend.list(uuid.uuid4()) == [] + + +def test_list_orders_oldest_first(backend: DbAPIKeyBackend, ws: uuid.UUID) -> None: + base = datetime.now(UTC) - timedelta(hours=1) + first = _record(ws, created_at=base) + second = _record(ws, created_at=base + timedelta(minutes=1)) + third = _record(ws, created_at=base + timedelta(minutes=2)) + for rec in (third, first, second): # insert out of order + backend.add(rec) + assert [r.id for r in backend.list(ws)] == [first.id, second.id, third.id] + + +# --------------------------------------------------------------------------- # +# get: workspace isolation # +# --------------------------------------------------------------------------- # + + +def test_get_respects_workspace( + backend: DbAPIKeyBackend, ws: uuid.UUID, other_ws: uuid.UUID +) -> None: + record = _record(ws) + backend.add(record) + assert backend.get(ws, record.id) is not None + assert backend.get(other_ws, record.id) is None # right id, wrong tenant + assert backend.get(ws, uuid.uuid4()) is None # unknown id + + +# --------------------------------------------------------------------------- # +# durability # +# --------------------------------------------------------------------------- # + + +def test_persists_across_backend_instances( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + record = _record(ws) + DbAPIKeyBackend(factory).add(record) + assert DbAPIKeyBackend(factory).get(ws, record.id) is not None + + +# --------------------------------------------------------------------------- # +# APIKeyStore end-to-end parity (mutation flows through the backend) # +# --------------------------------------------------------------------------- # + + +def test_store_mint_verify_stamps_last_used( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + store = APIKeyStore(secret_key=SECRET, backend=DbAPIKeyBackend(factory)) + info, token = store.mint(workspace_id=ws, name="ci", role=UserRole.MEMBER) + + verified = store.verify(token) + assert verified is not None + assert verified.id == info.id + # verify() stamps last_used_at by mutating the returned record; the DB backend + # write-throughs that so it is durable (not just on the transient object). + reread = DbAPIKeyBackend(factory).get(ws, info.id) + assert reread is not None and reread.last_used_at is not None + + +def test_store_verify_rejects_unknown_token( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + store = APIKeyStore(secret_key=SECRET, backend=DbAPIKeyBackend(factory)) + store.mint(workspace_id=ws, name="ci", role=UserRole.MEMBER) + assert store.verify("forge_sy_not-a-real-token") is None + + +def test_store_verify_rejects_expired( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + store = APIKeyStore(secret_key=SECRET, backend=DbAPIKeyBackend(factory)) + _, token = store.mint( + workspace_id=ws, + name="ci", + role=UserRole.MEMBER, + expires_at=datetime.now(UTC) - timedelta(seconds=1), + ) + assert store.verify(token) is None + + +def test_store_revoke_persists( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + store = APIKeyStore(secret_key=SECRET, backend=DbAPIKeyBackend(factory)) + info, token = store.mint(workspace_id=ws, name="ci", role=UserRole.MEMBER) + assert store.verify(token) is not None + + assert store.revoke(ws, info.id) is True + # Revocation flows through the write-through record → row.revoked_at set. + assert store.verify(token) is None + reread = DbAPIKeyBackend(factory).get(ws, info.id) + assert reread is not None and reread.is_active is False + # Idempotent-ish: revoking an unknown key is False. + assert store.revoke(ws, uuid.uuid4()) is False + + +def test_store_revoke_for_user_persists( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + user_id = uuid.uuid4() + with factory() as session: + session.add( + User(id=user_id, workspace_id=ws, email=f"u-{user_id.hex[:6]}@acme.dev", name="U") + ) + session.commit() + store = APIKeyStore(secret_key=SECRET, backend=DbAPIKeyBackend(factory)) + _, t1 = store.mint(workspace_id=ws, name="a", role=UserRole.MEMBER, user_id=user_id) + _, t2 = store.mint(workspace_id=ws, name="b", role=UserRole.MEMBER, user_id=user_id) + _, other = store.mint(workspace_id=ws, name="c", role=UserRole.MEMBER) + + assert store.revoke_for_user(ws, user_id) == 2 + assert store.verify(t1) is None + assert store.verify(t2) is None + assert store.verify(other) is not None # untouched + + +def test_store_list_keys_reflects_state( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + store = APIKeyStore(secret_key=SECRET, backend=DbAPIKeyBackend(factory)) + info, _ = store.mint(workspace_id=ws, name="ci", role=UserRole.MEMBER) + keys = store.list_keys(ws) + assert [k.id for k in keys] == [info.id] + assert keys[0].is_active is True + + store.revoke(ws, info.id) + assert store.list_keys(ws)[0].is_active is False + + +# --------------------------------------------------------------------------- # +# parity with the in-memory backend (same seam, identical observable results) # +# --------------------------------------------------------------------------- # + + +def test_matches_in_memory_backend_behaviour( + factory: sessionmaker[Session], ws: uuid.UUID +) -> None: + db = DbAPIKeyBackend(factory) + mem = InMemoryAPIKeyBackend() + + record = _record(ws, role=UserRole.VIEWER) + db.add(record) + mem.add(record) + + db_got = db.get(ws, record.id) + mem_got = mem.get(ws, record.id) + assert db_got is not None and mem_got is not None + assert (db_got.id, db_got.role, db_got.kind, db_got.key_prefix) == ( + mem_got.id, + mem_got.role, + mem_got.kind, + mem_got.key_prefix, + ) + assert [r.id for r in db.by_prefix(record.key_prefix)] == [ + r.id for r in mem.by_prefix(record.key_prefix) + ] + assert [r.id for r in db.list(ws)] == [r.id for r in mem.list(ws)] + assert db.get(uuid.uuid4(), record.id) is mem.get(uuid.uuid4(), record.id) # None From b3be6e7315310615190f27d5e16b4e229a86dcf0 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 14:30:18 +0200 Subject: [PATCH 06/12] feat(db/policy-audit-sink): Postgres persistence Co-Authored-By: Claude Fable 5 --- apps/api/forge_api/routers/policy.py | 10 +- .../services/policy_audit_sink_db.py | 98 +++++++ apps/api/forge_api/services/policy_service.py | 3 +- apps/api/forge_api/settings.py | 8 + apps/api/tests/test_policy_audit_sink_db.py | 273 ++++++++++++++++++ 5 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 apps/api/forge_api/services/policy_audit_sink_db.py create mode 100644 apps/api/tests/test_policy_audit_sink_db.py diff --git a/apps/api/forge_api/routers/policy.py b/apps/api/forge_api/routers/policy.py index ff27074e..c3c308bc 100644 --- a/apps/api/forge_api/routers/policy.py +++ b/apps/api/forge_api/routers/policy.py @@ -33,6 +33,7 @@ SimulateRequest, SimulationResult, ) +from forge_api.services.policy_audit_sink_db import build_policy_audit_sink from forge_api.services.policy_service import PolicyService from forge_contracts import Decision, Policy, ToolCall from forge_policy import ( @@ -74,7 +75,14 @@ def get_policy_evaluator() -> ConditionalPolicyEvaluator: @lru_cache(maxsize=1) def _policy_service_singleton() -> PolicyService: - return PolicyService(evaluator=_policy_evaluator_singleton()) + # The audit sink is env-selected (``FORGE_POLICY_AUDIT_BACKEND``): ``memory`` + # (default) keeps the hermetic in-memory sink; ``db`` durably persists each + # emitted ``policy.decision`` event to ``policy_rule_evaluation``. Both satisfy + # the same ``PolicyAuditSink`` seam, so the service is agnostic. + return PolicyService( + evaluator=_policy_evaluator_singleton(), + audit_sink=build_policy_audit_sink(), + ) def get_policy_service() -> PolicyService: diff --git a/apps/api/forge_api/services/policy_audit_sink_db.py b/apps/api/forge_api/services/policy_audit_sink_db.py new file mode 100644 index 00000000..1ea988ab --- /dev/null +++ b/apps/api/forge_api/services/policy_audit_sink_db.py @@ -0,0 +1,98 @@ +"""Postgres-backed policy-audit sink (policy-audit-sink persistence). + +:class:`DbPolicyAuditSink` is a drop-in, durable alternative to +:class:`~forge_api.services.policy_service.InMemoryPolicyAuditSink` that satisfies +the **same** :class:`~forge_api.services.policy_service.PolicyAuditSink` seam +(``emit(PolicyDecisionEvent)``) — so the F29 composition root swaps it in behind +``FORGE_POLICY_AUDIT_BACKEND=db`` with no behavioural change. The default stays +``memory`` and the in-memory sink remains the unit-test default. + +Where the in-memory sink appends each emitted ``policy.decision`` event to a +process-local list, this sink persists it durably: every ``emit`` opens its own +short unit-of-work and writes one append-only ``policy_rule_evaluation`` row (the +canonical, queryable F29 audit table created by migration 0011, hardened DB-side +by the F39 ``attach_immutability_trigger`` BEFORE UPDATE/DELETE block). The +compact, redacted :class:`~forge_api.services.policy_service.PolicyDecisionEvent` +maps field-for-field onto the row — it carries only the redacted projection +(never raw ``ToolCall.args`` / ``command``), so the durable trail inherits the +same F04/F10 redaction the event already guarantees. ``policy_snapshot_id`` and +the server-defaulted ``evaluated_at`` are the only row columns the event does not +carry (the event has no snapshot dimension; ``evaluated_at`` is stamped by the +database), matching how the in-memory sink also records neither. + +Because the sink owns its own ``sessionmaker`` (the ``emit`` seam receives no +session), the referential guarantees of ``policy_rule_evaluation`` are enforced by +the database exactly as for a directly-written row: a non-null ``agent_run_id`` +must resolve against the F07/F10 ``agent_run`` table or the insert is rejected. + +Note on composition (F29): the ``emit`` seam is an *independent* audit stream — +in this foundation no HTTP route invokes ``PolicyService.evaluate_and_record`` +(the only writer of the transactional row via a caller-supplied session), so +wiring this sink simply makes the emit stream itself durable. The default stays +``memory`` so every existing unit test is untouched. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from forge_api.services.policy_service import ( + InMemoryPolicyAuditSink, + PolicyAuditSink, + PolicyDecisionEvent, +) +from forge_db.models import PolicyRuleEvaluation + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["DbPolicyAuditSink", "build_policy_audit_sink"] + + +class DbPolicyAuditSink: + """A Postgres-backed policy-audit sink (implements ``PolicyAuditSink``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + def emit(self, event: PolicyDecisionEvent) -> None: + """Persist ``event`` as one append-only ``policy_rule_evaluation`` row.""" + with self._sf() as session: + session.add( + PolicyRuleEvaluation( + workspace_id=event.workspace_id, + agent_run_id=event.agent_run_id, + step_id=event.step_id, + action=event.action, + base_effect=event.base_effect, + final_effect=event.final_effect, + requires_approval=event.requires_approval, + severity=event.severity, + matched_rule_ids=list(event.matched_rule_ids), + context_redacted=dict(event.context_redacted), + ) + ) + session.commit() + + +# --------------------------------------------------------------------------- # +# Composition root # +# --------------------------------------------------------------------------- # + + +def build_policy_audit_sink() -> PolicyAuditSink: + """Return the process-wide policy-audit sink selected by ``FORGE_POLICY_AUDIT_BACKEND``. + + ``memory`` (default) → the hermetic + :class:`~forge_api.services.policy_service.InMemoryPolicyAuditSink` (unit-test + default, no Postgres); ``db`` → the durable :class:`DbPolicyAuditSink` bound to + the shared session factory. Both satisfy the same ``PolicyAuditSink`` seam, so + the ``PolicyService`` is agnostic to which is wired. + """ + from forge_api.settings import get_settings + + if get_settings().policy_audit_backend == "db": + from forge_api.db import get_session_factory + + return DbPolicyAuditSink(get_session_factory()) + return InMemoryPolicyAuditSink() diff --git a/apps/api/forge_api/services/policy_service.py b/apps/api/forge_api/services/policy_service.py index 42a506d0..b395fc83 100644 --- a/apps/api/forge_api/services/policy_service.py +++ b/apps/api/forge_api/services/policy_service.py @@ -21,7 +21,7 @@ from __future__ import annotations import uuid -from typing import Any, Protocol +from typing import Any, Protocol, runtime_checkable from pydantic import BaseModel, Field from sqlalchemy import select @@ -53,6 +53,7 @@ class PolicyDecisionEvent(BaseModel): step_id: uuid.UUID | None = None +@runtime_checkable class PolicyAuditSink(Protocol): """The audit seam F29 emits ``policy.decision`` events through.""" diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index b337b158..e49c4e25 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -134,6 +134,14 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # survive a restart. Read via ``FORGE_APIKEY_BACKEND``. apikey_backend: str = "memory" + # F29 policy-audit sink backend selection. ``memory`` (default) keeps the + # hermetic, process-memory ``InMemoryPolicyAuditSink`` (unit-test default, no + # Postgres); ``db`` wires the Postgres-backed ``DbPolicyAuditSink`` behind the + # same ``PolicyAuditSink`` seam so each emitted ``policy.decision`` event lands + # durably as an append-only ``policy_rule_evaluation`` row. Read via + # ``FORGE_POLICY_AUDIT_BACKEND``. + policy_audit_backend: str = "memory" + # F36 policy-override grant-store backend selection (J5). ``memory`` (default) # keeps the hermetic, process-memory ``InMemoryGrantStore`` (unit-test default, # no Postgres); ``db`` wires the Postgres-backed ``DbGrantStore`` behind the diff --git a/apps/api/tests/test_policy_audit_sink_db.py b/apps/api/tests/test_policy_audit_sink_db.py new file mode 100644 index 00000000..cdf86e00 --- /dev/null +++ b/apps/api/tests/test_policy_audit_sink_db.py @@ -0,0 +1,273 @@ +"""Postgres integration tests for :class:`DbPolicyAuditSink` (F29 policy-audit-sink). + +Exercises the DB-backed policy-audit sink against a real pgvector Postgres via the +shared ``pg_engine`` fixture (root ``conftest.py``): the ``emit`` seam end-to-end +— a full :class:`PolicyDecisionEvent` round-trip onto ``policy_rule_evaluation`` +(including the JSONB ``matched_rule_ids`` list + ``context_redacted`` dict), the +append-only accumulation + newest-first ordering the F29 audit query relies on, +workspace / agent-run filtering, the ``agent_run_id`` FK constraint (and the +null-run path), durability across sink instances, and structural + behavioural +parity with the in-memory sink both backends implement. + +Skips cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator + +import pytest +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.services.policy_audit_sink_db import DbPolicyAuditSink +from forge_api.services.policy_service import ( + InMemoryPolicyAuditSink, + PolicyAuditSink, + PolicyDecisionEvent, + PolicyService, +) +from forge_db.base import Base +from forge_db.models import AgentRun, PolicyRuleEvaluation, Workspace + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def seed(factory: sessionmaker[Session]) -> dict[str, uuid.UUID]: + """Two workspaces + two agent_runs (the ``policy_rule_evaluation`` FK targets).""" + ws = uuid.uuid4() + other_ws = uuid.uuid4() + run = uuid.uuid4() + other_run = uuid.uuid4() + with factory() as session: + session.add(Workspace(id=ws, name="Acme", slug=f"acme-{uuid.uuid4().hex[:8]}")) + session.add( + Workspace(id=other_ws, name="Other", slug=f"other-{uuid.uuid4().hex[:8]}") + ) + session.flush() + session.add(AgentRun(id=run, workspace_id=ws)) + session.add(AgentRun(id=other_run, workspace_id=other_ws)) + session.commit() + return {"ws": ws, "other_ws": other_ws, "run": run, "other_run": other_run} + + +@pytest.fixture +def sink(factory: sessionmaker[Session]) -> DbPolicyAuditSink: + return DbPolicyAuditSink(factory) + + +def _event( + seed: dict[str, uuid.UUID], + *, + workspace_id: uuid.UUID | None = None, + agent_run_id: uuid.UUID | None = "__default__", # type: ignore[assignment] + action: str = "write_file", + final_effect: str = "deny", + severity: str = "critical", + matched_rule_ids: list[str] | None = None, + step_id: uuid.UUID | None = None, +) -> PolicyDecisionEvent: + return PolicyDecisionEvent( + action=action, + base_effect="allow", + final_effect=final_effect, + requires_approval=True, + severity=severity, + matched_rule_ids=matched_rule_ids + if matched_rule_ids is not None + else ["infra-writes-main-only", "second-rule"], + context_redacted={"branch": "feature/x", "path": "infra/x.tf"}, + workspace_id=workspace_id or seed["ws"], + agent_run_id=seed["run"] if agent_run_id == "__default__" else agent_run_id, + step_id=step_id, + ) + + +def _rows(factory: sessionmaker[Session]) -> list[PolicyRuleEvaluation]: + with factory() as session: + return list( + session.execute( + select(PolicyRuleEvaluation).order_by( + PolicyRuleEvaluation.evaluated_at.asc() + ) + ) + .scalars() + .all() + ) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_sink_satisfies_policy_audit_sink_protocol(sink: DbPolicyAuditSink) -> None: + assert isinstance(sink, PolicyAuditSink) + # And it drops straight into the service seam the in-memory sink fills. + assert isinstance(PolicyService(audit_sink=sink).audit_sink, DbPolicyAuditSink) + + +# --------------------------------------------------------------------------- # +# emit round-trip # +# --------------------------------------------------------------------------- # + + +def test_emit_round_trips_every_field( + sink: DbPolicyAuditSink, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + step = uuid.uuid4() + sink.emit(_event(seed, step_id=step)) + + rows = _rows(factory) + assert len(rows) == 1 + row = rows[0] + assert row.workspace_id == seed["ws"] + assert row.agent_run_id == seed["run"] + assert row.step_id == step + assert row.action == "write_file" + assert row.base_effect == "allow" + assert row.final_effect == "deny" + assert row.requires_approval is True + assert row.severity == "critical" + # JSONB list + dict survive verbatim (order + contents preserved). + assert row.matched_rule_ids == ["infra-writes-main-only", "second-rule"] + assert row.context_redacted == {"branch": "feature/x", "path": "infra/x.tf"} + # Server-default timestamp populated; nothing beyond the event was written. + assert row.evaluated_at is not None + assert row.policy_snapshot_id is None + + +def test_emit_persists_null_agent_run( + sink: DbPolicyAuditSink, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + sink.emit(_event(seed, agent_run_id=None)) + rows = _rows(factory) + assert len(rows) == 1 + assert rows[0].agent_run_id is None + + +# --------------------------------------------------------------------------- # +# append-only accumulation + ordering # +# --------------------------------------------------------------------------- # + + +def test_emit_is_append_only_and_newest_first_query( + sink: DbPolicyAuditSink, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + for effect in ("deny", "allow", "deny"): + sink.emit(_event(seed, final_effect=effect)) + # Three distinct append-only rows — never an update. + assert len(_rows(factory)) == 3 + + # The F29 workspace query (newest-first) reads them back in evaluated_at desc. + service = PolicyService(audit_sink=sink) + with factory() as session: + listed = service.list_rule_evaluations(session, workspace_id=seed["ws"]) + assert len(listed) == 3 + stamps = [r.evaluated_at for r in listed] + assert stamps == sorted(stamps, reverse=True) + + +# --------------------------------------------------------------------------- # +# workspace + agent-run filtering # +# --------------------------------------------------------------------------- # + + +def test_query_is_workspace_and_run_scoped( + sink: DbPolicyAuditSink, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + sink.emit(_event(seed)) # ws / run + sink.emit(_event(seed)) # ws / run + sink.emit(_event(seed, workspace_id=seed["other_ws"], agent_run_id=seed["other_run"])) + + service = PolicyService(audit_sink=sink) + with factory() as session: + ws_rows = service.list_rule_evaluations(session, workspace_id=seed["ws"]) + other_rows = service.list_rule_evaluations(session, workspace_id=seed["other_ws"]) + by_run = service.list_rule_evaluations( + session, workspace_id=seed["ws"], agent_run_id=seed["run"] + ) + foreign_run = service.list_rule_evaluations( + session, workspace_id=seed["ws"], agent_run_id=seed["other_run"] + ) + assert len(ws_rows) == 2 + assert len(other_rows) == 1 + assert len(by_run) == 2 + assert foreign_run == [] + + +# --------------------------------------------------------------------------- # +# FK constraint (referential integrity enforced by the database) # +# --------------------------------------------------------------------------- # + + +def test_emit_unknown_agent_run_rejected( + sink: DbPolicyAuditSink, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + ghost = _event(seed, agent_run_id=uuid.uuid4()) + with pytest.raises(IntegrityError): + sink.emit(ghost) + # The failed insert left no row behind, and the sink still works afterwards. + assert _rows(factory) == [] + sink.emit(_event(seed)) + assert len(_rows(factory)) == 1 + + +# --------------------------------------------------------------------------- # +# durability across sink instances # +# --------------------------------------------------------------------------- # + + +def test_persists_across_sink_instances( + seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + DbPolicyAuditSink(factory).emit(_event(seed)) + # A second, independently constructed sink sees the same durable trail. + rows = _rows(factory) + assert len(rows) == 1 + assert rows[0].matched_rule_ids == ["infra-writes-main-only", "second-rule"] + + +# --------------------------------------------------------------------------- # +# parity with the in-memory sink (same seam, identical projection) # +# --------------------------------------------------------------------------- # + + +def test_matches_in_memory_sink_projection( + sink: DbPolicyAuditSink, seed: dict[str, uuid.UUID], factory: sessionmaker[Session] +) -> None: + mem = InMemoryPolicyAuditSink() + event = _event(seed) + + mem.emit(event) + sink.emit(event) + + assert len(mem.events) == 1 + captured = mem.events[0] + + rows = _rows(factory) + assert len(rows) == 1 + row = rows[0] + # The durable row is the exact projection the in-memory sink captured. + assert row.action == captured.action + assert row.base_effect == captured.base_effect + assert row.final_effect == captured.final_effect + assert row.requires_approval == captured.requires_approval + assert row.severity == captured.severity + assert row.matched_rule_ids == captured.matched_rule_ids + assert row.context_redacted == captured.context_redacted + assert row.workspace_id == captured.workspace_id + assert row.agent_run_id == captured.agent_run_id From 04f29b1eed3370465e246f5ac1a5df817c9b9eb7 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 15:20:17 +0200 Subject: [PATCH 07/12] feat(db/spec-projection-repository): Postgres persistence Co-Authored-By: Claude Fable 5 --- .../services/projection_repository_db.py | 319 ++++++++++++ .../forge_api/services/projection_service.py | 38 ++ apps/api/forge_api/settings.py | 9 + .../tests/test_projection_repository_db.py | 488 ++++++++++++++++++ apps/api/tests/test_projection_service.py | 47 ++ 5 files changed, 901 insertions(+) create mode 100644 apps/api/forge_api/services/projection_repository_db.py create mode 100644 apps/api/forge_api/services/projection_service.py create mode 100644 apps/api/tests/test_projection_repository_db.py create mode 100644 apps/api/tests/test_projection_service.py diff --git a/apps/api/forge_api/services/projection_repository_db.py b/apps/api/forge_api/services/projection_repository_db.py new file mode 100644 index 00000000..30a29df7 --- /dev/null +++ b/apps/api/forge_api/services/projection_repository_db.py @@ -0,0 +1,319 @@ +"""Postgres-backed :class:`~forge_spec.projection.ProjectionRepository` (F23). + +:class:`SqlAlchemyProjectionRepository` is a drop-in, durable alternative to +:class:`~forge_spec.projection.InMemoryProjectionRepository` that satisfies the +**same** ``ProjectionRepository`` protocol (``replace_spec_links`` / +``upsert_rollup`` / ``get_rollup`` / ``get_projection_version`` / +``list_rollups`` / ``get_links`` / ``list_links``) — so the F23 composition root +swaps it in behind ``FORGE_PROJECTION_BACKEND=db`` with no behavioural change. +The default stays ``memory`` and the in-memory store remains the unit-test +default (the projection class docstring calls the DB repo the "parked" sync repo; +this is that repo, now built). + +It lives in ``apps/api`` (not the ``forge_spec`` package, which is deliberately +Protocol-ported and DB-free — the module docstring says "a thin adapter wires the +real sync-SQLAlchemy foundation in ``apps/``"), exactly like the sibling +:class:`~forge_api.services.approval_repository_db.SqlAlchemyApprovalRepository`. +It maps the two projection DTOs +(:class:`~forge_spec.dashboard_schemas.CriterionLinkRecord` / +:class:`~forge_spec.dashboard_schemas.SpecRollupRecord`) onto the canonical F23 +ORM rows (``forge_db.models.TraceabilityCriterionLink`` / +``TraceabilitySpecRollup``, migration ``0014``). + +Behaviour parity with the in-memory store is exact and intentional: + +* ``replace_spec_links`` rewrites a spec's link rows **wholesale** (delete all + rows for the spec, then insert the new set) — the DB analogue of the in-memory + ``self._links[spec_id] = list(links)``; +* ``upsert_rollup`` bumps a **monotonic** ``projection_version`` per spec: it + ``SELECT ... FOR UPDATE`` locks the existing rollup row so concurrent refreshes + serialize (the DB analogue of the in-memory GIL-guarded ``+= 1``), returns ``1`` + on the first upsert and ``old + 1`` thereafter — never reused, never regressed; +* ``get_projection_version`` reads the persisted counter (``0`` when the spec has + no rollup yet, matching the in-memory default); +* reads filter by ``spec_id`` / ``project_id`` only (the protocol carries no + workspace dimension, mirroring the in-memory store's flat maps) and return + deterministically ordered rows. + +Storage-boundary divergences (shared with every DB-backed repo here; both still +satisfy the same protocol): + +* the ids the protocol types as ``str`` are real ``uuid`` columns, so a write + requires canonical UUID strings and an existing ``workspace`` / ``project`` / + ``spec_document`` parent (the FKs are real); a read for a non-UUID or absent id + reads as "absent" (``None`` / ``[]`` / ``0``), exactly as the in-memory store + returns for an unknown key; +* the ``(spec_id, criterion_ext_id)`` and ``(spec_id)`` unique constraints are + enforced by the database — the wholesale rewrite/upsert never trips them in the + projector's normal flow, but a direct duplicate insert raises at the boundary + rather than silently overwriting; +* the coverage ratios land in ``NUMERIC(5, 4)`` columns, so they round-trip to + four decimal places (the projector's ratios are already rounded to that grain). +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from sqlalchemy import delete, select + +from forge_db.models import TraceabilityCriterionLink, TraceabilitySpecRollup +from forge_spec.dashboard_schemas import CriterionLinkRecord, SpecRollupRecord + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["SqlAlchemyProjectionRepository"] + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _aware(value: datetime | None) -> datetime | None: + """Normalise a stored timestamp to timezone-aware UTC (SQLite reads naive).""" + if value is None: + return None + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +def _maybe_uuid(value: str | None) -> uuid.UUID | None: + """Parse ``value`` as a UUID, or ``None`` — a read for a non-UUID id is absent.""" + if value is None: + return None + try: + return uuid.UUID(str(value)) + except (ValueError, AttributeError, TypeError): + return None + + +def _link_record(row: TraceabilityCriterionLink) -> CriterionLinkRecord: + return CriterionLinkRecord( + workspace_id=str(row.workspace_id), + project_id=str(row.project_id), + spec_id=str(row.spec_id), + spec_key=row.spec_key, + criterion_ext_id=row.criterion_ext_id, + criterion_text=row.criterion_text, + requirement_ext_ids=list(row.requirement_ext_ids or []), + status=row.status, # str -> CellStatus (pydantic coercion) + satisfied=row.satisfied, + test_refs=list(row.test_refs or []), + diff_refs=list(row.diff_refs or []), + task_ids=list(row.task_ids or []), + pr_numbers=list(row.pr_numbers or []), + report_spec_version=row.report_spec_version, + current_spec_version=row.current_spec_version, + last_validated_at=_aware(row.last_validated_at), + ) + + +def _rollup_record(row: TraceabilitySpecRollup) -> SpecRollupRecord: + return SpecRollupRecord( + workspace_id=str(row.workspace_id), + project_id=str(row.project_id), + spec_id=str(row.spec_id), + spec_key=row.spec_key, + spec_name=row.spec_name, + epic_id=str(row.epic_id) if row.epic_id is not None else None, + spec_status=row.spec_status, + total_requirements=row.total_requirements, + covered_requirements=row.covered_requirements, + total_criteria=row.total_criteria, + validated_criteria=row.validated_criteria, + failed_criteria=row.failed_criteria, + uncovered_criteria=row.uncovered_criteria, + claimed_criteria=row.claimed_criteria, + stale_criteria=row.stale_criteria, + requirement_coverage=float(row.requirement_coverage), + acceptance_criteria_coverage=float(row.acceptance_criteria_coverage), + uncovered_requirement_ext_ids=list(row.uncovered_requirement_ext_ids or []), + validation_status=row.validation_status, # str -> ValidationStatus + gap_count=row.gap_count, + last_validated_at=_aware(row.last_validated_at), + ) + + +class SqlAlchemyProjectionRepository: + """A Postgres-backed ``ProjectionRepository`` (F23 traceability projection).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + @contextmanager + def _session(self) -> Iterator[Session]: + """A session that commits on success and always closes (writes + reads).""" + session = self._sf() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + # -- writes -------------------------------------------------------------- # + + def replace_spec_links(self, spec_id: str, links: list[CriterionLinkRecord]) -> None: + """Rewrite ``spec_id``'s link rows wholesale (delete-all, then insert).""" + spec_uuid = uuid.UUID(spec_id) + now = _now() + with self._session() as session: + session.execute( + delete(TraceabilityCriterionLink).where( + TraceabilityCriterionLink.spec_id == spec_uuid + ) + ) + session.flush() + for link in links: + session.add( + TraceabilityCriterionLink( + workspace_id=uuid.UUID(link.workspace_id), + project_id=uuid.UUID(link.project_id), + spec_id=uuid.UUID(link.spec_id), + spec_key=link.spec_key, + criterion_ext_id=link.criterion_ext_id, + criterion_text=link.criterion_text, + requirement_ext_ids=list(link.requirement_ext_ids), + status=link.status.value, + satisfied=link.satisfied, + test_refs=list(link.test_refs), + diff_refs=list(link.diff_refs), + task_ids=list(link.task_ids), + pr_numbers=list(link.pr_numbers), + report_spec_version=link.report_spec_version, + current_spec_version=link.current_spec_version, + last_validated_at=link.last_validated_at, + refreshed_at=now, + ) + ) + + def upsert_rollup(self, rollup: SpecRollupRecord) -> int: + """Upsert the rollup row, bump ``projection_version`` monotonically, return it.""" + spec_uuid = uuid.UUID(rollup.spec_id) + with self._session() as session: + row = session.execute( + select(TraceabilitySpecRollup) + .where(TraceabilitySpecRollup.spec_id == spec_uuid) + .with_for_update() + ).scalar_one_or_none() + if row is None: + new_version = 1 + row = TraceabilitySpecRollup(spec_id=spec_uuid) + session.add(row) + else: + new_version = row.projection_version + 1 + self._apply_rollup(row, rollup, new_version) + return new_version + + @staticmethod + def _apply_rollup( + row: TraceabilitySpecRollup, rollup: SpecRollupRecord, version: int + ) -> None: + row.workspace_id = uuid.UUID(rollup.workspace_id) + row.project_id = uuid.UUID(rollup.project_id) + row.spec_id = uuid.UUID(rollup.spec_id) + row.spec_key = rollup.spec_key + row.spec_name = rollup.spec_name + row.epic_id = uuid.UUID(rollup.epic_id) if rollup.epic_id else None + row.spec_status = rollup.spec_status + row.total_requirements = rollup.total_requirements + row.covered_requirements = rollup.covered_requirements + row.total_criteria = rollup.total_criteria + row.validated_criteria = rollup.validated_criteria + row.failed_criteria = rollup.failed_criteria + row.uncovered_criteria = rollup.uncovered_criteria + row.claimed_criteria = rollup.claimed_criteria + row.stale_criteria = rollup.stale_criteria + row.requirement_coverage = rollup.requirement_coverage + row.acceptance_criteria_coverage = rollup.acceptance_criteria_coverage + row.uncovered_requirement_ext_ids = list(rollup.uncovered_requirement_ext_ids) + row.validation_status = rollup.validation_status.value + row.gap_count = rollup.gap_count + row.last_validated_at = rollup.last_validated_at + row.projection_version = version + row.refreshed_at = _now() + + # -- reads --------------------------------------------------------------- # + + def get_rollup(self, spec_id: str) -> SpecRollupRecord | None: + spec_uuid = _maybe_uuid(spec_id) + if spec_uuid is None: + return None + with self._session() as session: + row = session.execute( + select(TraceabilitySpecRollup).where( + TraceabilitySpecRollup.spec_id == spec_uuid + ) + ).scalar_one_or_none() + return _rollup_record(row) if row is not None else None + + def get_projection_version(self, spec_id: str) -> int: + spec_uuid = _maybe_uuid(spec_id) + if spec_uuid is None: + return 0 + with self._session() as session: + version = session.execute( + select(TraceabilitySpecRollup.projection_version).where( + TraceabilitySpecRollup.spec_id == spec_uuid + ) + ).scalar_one_or_none() + return int(version) if version is not None else 0 + + def list_rollups(self, project_id: str) -> list[SpecRollupRecord]: + project_uuid = _maybe_uuid(project_id) + if project_uuid is None: + return [] + with self._session() as session: + rows = ( + session.execute( + select(TraceabilitySpecRollup) + .where(TraceabilitySpecRollup.project_id == project_uuid) + .order_by( + TraceabilitySpecRollup.spec_key, TraceabilitySpecRollup.spec_id + ) + ) + .scalars() + .all() + ) + return [_rollup_record(row) for row in rows] + + def get_links(self, spec_id: str) -> list[CriterionLinkRecord]: + spec_uuid = _maybe_uuid(spec_id) + if spec_uuid is None: + return [] + with self._session() as session: + rows = ( + session.execute( + select(TraceabilityCriterionLink) + .where(TraceabilityCriterionLink.spec_id == spec_uuid) + .order_by(TraceabilityCriterionLink.criterion_ext_id) + ) + .scalars() + .all() + ) + return [_link_record(row) for row in rows] + + def list_links(self, project_id: str) -> list[CriterionLinkRecord]: + project_uuid = _maybe_uuid(project_id) + if project_uuid is None: + return [] + with self._session() as session: + rows = ( + session.execute( + select(TraceabilityCriterionLink) + .where(TraceabilityCriterionLink.project_id == project_uuid) + .order_by( + TraceabilityCriterionLink.spec_key, + TraceabilityCriterionLink.criterion_ext_id, + ) + ) + .scalars() + .all() + ) + return [_link_record(row) for row in rows] diff --git a/apps/api/forge_api/services/projection_service.py b/apps/api/forge_api/services/projection_service.py new file mode 100644 index 00000000..323f550a --- /dev/null +++ b/apps/api/forge_api/services/projection_service.py @@ -0,0 +1,38 @@ +"""F23 composition root: the traceability-projection repository seam. + +:func:`build_projection_repository` selects the ``ProjectionRepository`` backend +via ``FORGE_PROJECTION_BACKEND`` (default ``memory``), exactly like the sibling +``FORGE_BOARD_BACKEND`` / ``FORGE_APPROVAL_BACKEND`` seams: + +* ``memory`` (default) -> the hermetic + :class:`~forge_spec.projection.InMemoryProjectionRepository` (the unit-test + default; no Postgres, so every existing spec-engine test stays green untouched); +* ``db`` -> the durable + :class:`~forge_api.services.projection_repository_db.SqlAlchemyProjectionRepository` + bound to the shared session factory. + +Both satisfy the same ``ProjectionRepository`` protocol, so the swap is +behaviour-preserving: the ``TraceabilityProjector`` / ``DashboardService`` read +and write through the port and never learn which backend they got. The DB import +is deferred so the default path never imports ``forge_db`` / opens a connection. +""" + +from __future__ import annotations + +from forge_spec import InMemoryProjectionRepository, ProjectionRepository + +__all__ = ["build_projection_repository"] + + +def build_projection_repository() -> ProjectionRepository: + """Return the projection repository selected by ``FORGE_PROJECTION_BACKEND``.""" + from forge_api.settings import get_settings + + if get_settings().projection_backend == "db": + from forge_api.db import get_session_factory + from forge_api.services.projection_repository_db import ( + SqlAlchemyProjectionRepository, + ) + + return SqlAlchemyProjectionRepository(get_session_factory()) + return InMemoryProjectionRepository() diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index e49c4e25..6666b71c 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -134,6 +134,15 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # survive a restart. Read via ``FORGE_APIKEY_BACKEND``. apikey_backend: str = "memory" + # F23 traceability-projection repository backend selection. ``memory`` + # (default) keeps the hermetic, process-memory ``InMemoryProjectionRepository`` + # (unit-test default, no Postgres); ``db`` wires the Postgres-backed + # ``SqlAlchemyProjectionRepository`` behind the same ``ProjectionRepository`` + # protocol so the F23 dashboard's denormalised projection (criterion links + + # spec rollups, with the monotonic ``projection_version``) is durably + # persisted. Read via ``FORGE_PROJECTION_BACKEND``. + projection_backend: str = "memory" + # F29 policy-audit sink backend selection. ``memory`` (default) keeps the # hermetic, process-memory ``InMemoryPolicyAuditSink`` (unit-test default, no # Postgres); ``db`` wires the Postgres-backed ``DbPolicyAuditSink`` behind the diff --git a/apps/api/tests/test_projection_repository_db.py b/apps/api/tests/test_projection_repository_db.py new file mode 100644 index 00000000..0c572c7e --- /dev/null +++ b/apps/api/tests/test_projection_repository_db.py @@ -0,0 +1,488 @@ +"""Postgres integration tests for :class:`SqlAlchemyProjectionRepository` (F23). + +Exercises the DB-backed traceability-projection repository against a real +pgvector Postgres via the shared ``pg_engine`` fixture (root ``conftest.py``): +the full ``ProjectionRepository`` protocol end-to-end — a wholesale link-set +round-trip (every ``CriterionLinkRecord`` field), the monotonic +``projection_version`` bump (``SELECT ... FOR UPDATE`` upsert), a full +``SpecRollupRecord`` round-trip (coverage ratios, enum status, epic id, +timestamp), project-scoped ``list_rollups`` / ``list_links`` filtering + ordering, +the ``(spec_id)`` / ``(spec_id, criterion_ext_id)`` unique constraints, durability +across repository instances, byte-for-byte parity with the in-memory store, and +structural conformance to the same protocol the in-memory store implements. Skips +cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. + +Each behaviour mirrors the in-memory contract, so both backends are proven to +satisfy the same protocol identically. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime + +import pytest +from sqlalchemy import func, select +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.services.projection_repository_db import SqlAlchemyProjectionRepository +from forge_db.base import Base +from forge_db.models import ( + Project, + SpecDocument, + TraceabilitySpecRollup, + Workspace, +) +from forge_spec import InMemoryProjectionRepository, ProjectionRepository +from forge_spec.dashboard_schemas import ( + CellStatus, + CriterionLinkRecord, + SpecRollupRecord, + ValidationStatus, +) + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + +_PROTOCOL_METHODS = ( + "replace_spec_links", + "upsert_rollup", + "get_rollup", + "get_projection_version", + "list_rollups", + "get_links", + "list_links", +) + +_VALIDATED_AT = datetime(2026, 2, 3, 4, 5, 6, tzinfo=UTC) + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def repo(factory: sessionmaker[Session]) -> SqlAlchemyProjectionRepository: + return SqlAlchemyProjectionRepository(factory) + + +class _Ids: + """A seeded workspace with two projects, each carrying one spec_document.""" + + def __init__(self) -> None: + self.ws = uuid.uuid4() + self.proj_a = uuid.uuid4() + self.proj_b = uuid.uuid4() + self.spec_a1 = uuid.uuid4() + self.spec_a2 = uuid.uuid4() + self.spec_b1 = uuid.uuid4() + + +@pytest.fixture +def ids(factory: sessionmaker[Session]) -> _Ids: + seeded = _Ids() + with factory() as session: + session.add( + Workspace(id=seeded.ws, name="Acme", slug=f"acme-{uuid.uuid4().hex[:8]}") + ) + session.flush() + for pid, key in ((seeded.proj_a, "PA"), (seeded.proj_b, "PB")): + session.add( + Project(id=pid, workspace_id=seeded.ws, name=f"Project {key}", key=key) + ) + session.flush() + specs = ( + (seeded.spec_a1, seeded.proj_a, "SPEC-A1"), + (seeded.spec_a2, seeded.proj_a, "SPEC-A2"), + (seeded.spec_b1, seeded.proj_b, "SPEC-B1"), + ) + for sid, pid, key in specs: + session.add( + SpecDocument( + id=sid, + workspace_id=seeded.ws, + project_id=pid, + spec_key=key, + name=f"{key} name", + ) + ) + session.commit() + return seeded + + +def _link( + ids: _Ids, + *, + spec_id: uuid.UUID, + project_id: uuid.UUID, + spec_key: str, + criterion: str, + status: CellStatus = CellStatus.VALIDATED, + **kw, +) -> CriterionLinkRecord: + base = { + "workspace_id": str(ids.ws), + "project_id": str(project_id), + "spec_id": str(spec_id), + "spec_key": spec_key, + "criterion_ext_id": criterion, + "criterion_text": f"{criterion} text", + "status": status, + } + base.update(kw) + return CriterionLinkRecord(**base) + + +def _lk( + ids: _Ids, sid: uuid.UUID, pid: uuid.UUID, key: str, crit: str, **kw +) -> CriterionLinkRecord: + """Positional shorthand for :func:`_link` (keeps list literals within width).""" + return _link(ids, spec_id=sid, project_id=pid, spec_key=key, criterion=crit, **kw) + + +def _rollup( + ids: _Ids, + *, + spec_id: uuid.UUID, + project_id: uuid.UUID, + spec_key: str, + **kw, +) -> SpecRollupRecord: + base = { + "workspace_id": str(ids.ws), + "project_id": str(project_id), + "spec_id": str(spec_id), + "spec_key": spec_key, + "spec_name": f"{spec_key} name", + "spec_status": "active", + "total_requirements": 4, + "covered_requirements": 3, + "total_criteria": 4, + "validated_criteria": 2, + "failed_criteria": 1, + "uncovered_criteria": 1, + "claimed_criteria": 0, + "stale_criteria": 0, + "requirement_coverage": 0.75, + "acceptance_criteria_coverage": 0.5, + "validation_status": ValidationStatus.PARTIAL, + "gap_count": 2, + } + base.update(kw) + return SpecRollupRecord(**base) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_repo_satisfies_projection_repository_protocol( + repo: SqlAlchemyProjectionRepository, +) -> None: + for name in _PROTOCOL_METHODS: + assert callable(getattr(repo, name)), name + # A structural stand-in for the (non-runtime-checkable) Protocol: the DB repo + # is assignable where a ``ProjectionRepository`` is expected. + port: ProjectionRepository = repo + assert port is repo + + +# --------------------------------------------------------------------------- # +# Links: wholesale round-trip + replace semantics # +# --------------------------------------------------------------------------- # + + +def test_replace_and_get_links_roundtrip( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + link = _link( + ids, + spec_id=ids.spec_a1, + project_id=ids.proj_a, + spec_key="SPEC-A1", + criterion="A1", + status=CellStatus.VALIDATED, + requirement_ext_ids=["R1", "R3"], + satisfied=True, + test_refs=["t::one", "t::two"], + diff_refs=["d1"], + task_ids=["TASK-1"], + pr_numbers=[7, 9], + report_spec_version=4, + current_spec_version=5, + last_validated_at=_VALIDATED_AT, + ) + repo.replace_spec_links(str(ids.spec_a1), [link]) + + got = repo.get_links(str(ids.spec_a1)) + assert len(got) == 1 + assert got[0] == link + + +def test_replace_spec_links_is_wholesale( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + first = [ + _lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", c) + for c in ("A1", "A2") + ] + repo.replace_spec_links(str(ids.spec_a1), first) + assert [link_.criterion_ext_id for link_ in repo.get_links(str(ids.spec_a1))] == ["A1", "A2"] + + # Wholesale replace: the old set is gone, only the new row remains. + second = [ + _lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", "A3") + ] + repo.replace_spec_links(str(ids.spec_a1), second) + assert [link_.criterion_ext_id for link_ in repo.get_links(str(ids.spec_a1))] == ["A3"] + + +def test_replace_with_empty_clears_links( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + repo.replace_spec_links( + str(ids.spec_a1), + [_lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", "A1")], + ) + repo.replace_spec_links(str(ids.spec_a1), []) + assert repo.get_links(str(ids.spec_a1)) == [] + + +def test_get_links_ordered_by_criterion( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + unordered = [ + _lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", c) + for c in ("A3", "A1", "A2") + ] + repo.replace_spec_links(str(ids.spec_a1), unordered) + assert [link_.criterion_ext_id for link_ in repo.get_links(str(ids.spec_a1))] == [ + "A1", + "A2", + "A3", + ] + + +def test_get_links_absent_or_non_uuid(repo: SqlAlchemyProjectionRepository) -> None: + assert repo.get_links(str(uuid.uuid4())) == [] + assert repo.get_links("not-a-uuid") == [] + + +# --------------------------------------------------------------------------- # +# Rollup: round-trip + monotonic projection_version # +# --------------------------------------------------------------------------- # + + +def test_upsert_rollup_bumps_version_monotonically( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + rollup = _rollup(ids, spec_id=ids.spec_a1, project_id=ids.proj_a, spec_key="SPEC-A1") + assert repo.get_projection_version(str(ids.spec_a1)) == 0 + + assert repo.upsert_rollup(rollup) == 1 + assert repo.upsert_rollup(rollup) == 2 + assert repo.upsert_rollup(rollup) == 3 + assert repo.get_projection_version(str(ids.spec_a1)) == 3 + + +def test_upsert_rollup_keeps_single_row( + repo: SqlAlchemyProjectionRepository, ids: _Ids, factory: sessionmaker[Session] +) -> None: + rollup = _rollup(ids, spec_id=ids.spec_a1, project_id=ids.proj_a, spec_key="SPEC-A1") + repo.upsert_rollup(rollup) + repo.upsert_rollup(rollup) + with factory() as session: + count = session.execute( + select(func.count()) + .select_from(TraceabilitySpecRollup) + .where(TraceabilitySpecRollup.spec_id == ids.spec_a1) + ).scalar_one() + assert count == 1 + + +def test_rollup_full_field_roundtrip( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + epic_id = str(uuid.uuid4()) + rollup = _rollup( + ids, + spec_id=ids.spec_a1, + project_id=ids.proj_a, + spec_key="SPEC-A1", + epic_id=epic_id, + uncovered_requirement_ext_ids=["R2", "R4"], + validation_status=ValidationStatus.FAILING, + requirement_coverage=0.25, + acceptance_criteria_coverage=1.0, + last_validated_at=_VALIDATED_AT, + ) + repo.upsert_rollup(rollup) + + got = repo.get_rollup(str(ids.spec_a1)) + assert got == rollup + assert got is not None + assert got.epic_id == epic_id + assert got.validation_status is ValidationStatus.FAILING + assert got.uncovered_requirement_ext_ids == ["R2", "R4"] + + +def test_get_rollup_and_version_absent(repo: SqlAlchemyProjectionRepository) -> None: + assert repo.get_rollup(str(uuid.uuid4())) is None + assert repo.get_rollup("not-a-uuid") is None + assert repo.get_projection_version(str(uuid.uuid4())) == 0 + assert repo.get_projection_version("not-a-uuid") == 0 + + +def test_upsert_updates_row_data( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + repo.upsert_rollup( + _rollup(ids, spec_id=ids.spec_a1, project_id=ids.proj_a, spec_key="SPEC-A1", gap_count=2) + ) + repo.upsert_rollup( + _rollup( + ids, + spec_id=ids.spec_a1, + project_id=ids.proj_a, + spec_key="SPEC-A1", + gap_count=0, + validation_status=ValidationStatus.PASSING, + ) + ) + got = repo.get_rollup(str(ids.spec_a1)) + assert got is not None + assert got.gap_count == 0 + assert got.validation_status is ValidationStatus.PASSING + + +# --------------------------------------------------------------------------- # +# Project-scoped list reads: filtering + ordering # +# --------------------------------------------------------------------------- # + + +def test_list_rollups_filters_by_project_and_orders( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + # proj_a has two specs (seeded out of key order), proj_b has one. + repo.upsert_rollup( + _rollup(ids, spec_id=ids.spec_a2, project_id=ids.proj_a, spec_key="SPEC-A2") + ) + repo.upsert_rollup( + _rollup(ids, spec_id=ids.spec_a1, project_id=ids.proj_a, spec_key="SPEC-A1") + ) + repo.upsert_rollup( + _rollup(ids, spec_id=ids.spec_b1, project_id=ids.proj_b, spec_key="SPEC-B1") + ) + + rollups_a = repo.list_rollups(str(ids.proj_a)) + assert [r.spec_key for r in rollups_a] == ["SPEC-A1", "SPEC-A2"] # ordered by spec_key + assert {r.project_id for r in rollups_a} == {str(ids.proj_a)} + + rollups_b = repo.list_rollups(str(ids.proj_b)) + assert [r.spec_key for r in rollups_b] == ["SPEC-B1"] + + assert repo.list_rollups(str(uuid.uuid4())) == [] + assert repo.list_rollups("not-a-uuid") == [] + + +def test_list_links_filters_by_project_and_orders( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + repo.replace_spec_links( + str(ids.spec_a2), + [_lk(ids, ids.spec_a2, ids.proj_a, "SPEC-A2", "B1")], + ) + repo.replace_spec_links( + str(ids.spec_a1), + [ + _lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", "A2"), + _lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", "A1"), + ], + ) + repo.replace_spec_links( + str(ids.spec_b1), + [_lk(ids, ids.spec_b1, ids.proj_b, "SPEC-B1", "A1")], + ) + + links_a = repo.list_links(str(ids.proj_a)) + # Ordered by (spec_key, criterion_ext_id); only proj_a's links. + assert [(link_.spec_key, link_.criterion_ext_id) for link_ in links_a] == [ + ("SPEC-A1", "A1"), + ("SPEC-A1", "A2"), + ("SPEC-A2", "B1"), + ] + assert {link_.project_id for link_ in links_a} == {str(ids.proj_a)} + + assert [link_.spec_key for link_ in repo.list_links(str(ids.proj_b))] == ["SPEC-B1"] + assert repo.list_links("not-a-uuid") == [] + + +# --------------------------------------------------------------------------- # +# Durability + parity # +# --------------------------------------------------------------------------- # + + +def test_durable_across_repository_instances( + factory: sessionmaker[Session], ids: _Ids +) -> None: + writer = SqlAlchemyProjectionRepository(factory) + writer.replace_spec_links( + str(ids.spec_a1), + [_lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", "A1")], + ) + version = writer.upsert_rollup( + _rollup(ids, spec_id=ids.spec_a1, project_id=ids.proj_a, spec_key="SPEC-A1") + ) + + reader = SqlAlchemyProjectionRepository(factory) + assert reader.get_projection_version(str(ids.spec_a1)) == version + assert reader.get_rollup(str(ids.spec_a1)) is not None + assert [link_.criterion_ext_id for link_ in reader.get_links(str(ids.spec_a1))] == ["A1"] + + +def test_parity_with_in_memory_store( + repo: SqlAlchemyProjectionRepository, ids: _Ids +) -> None: + mem: ProjectionRepository = InMemoryProjectionRepository() + links = [ + _link( + ids, + spec_id=ids.spec_a1, + project_id=ids.proj_a, + spec_key="SPEC-A1", + criterion="A1", + satisfied=True, + test_refs=["t::a"], + pr_numbers=[3], + last_validated_at=_VALIDATED_AT, + ), + _lk(ids, ids.spec_a1, ids.proj_a, "SPEC-A1", "A2"), + ] + rollup = _rollup( + ids, + spec_id=ids.spec_a1, + project_id=ids.proj_a, + spec_key="SPEC-A1", + last_validated_at=_VALIDATED_AT, + ) + for backend in (repo, mem): + backend.replace_spec_links(str(ids.spec_a1), links) + v1 = backend.upsert_rollup(rollup) + v2 = backend.upsert_rollup(rollup) + assert (v1, v2) == (1, 2) + + assert repo.get_projection_version(str(ids.spec_a1)) == mem.get_projection_version( + str(ids.spec_a1) + ) + assert repo.get_rollup(str(ids.spec_a1)) == mem.get_rollup(str(ids.spec_a1)) + assert sorted(repo.get_links(str(ids.spec_a1)), key=lambda link_: link_.criterion_ext_id) == ( + sorted(mem.get_links(str(ids.spec_a1)), key=lambda link_: link_.criterion_ext_id) + ) + assert repo.list_rollups(str(ids.proj_a)) == mem.list_rollups(str(ids.proj_a)) diff --git a/apps/api/tests/test_projection_service.py b/apps/api/tests/test_projection_service.py new file mode 100644 index 00000000..5aaa14f2 --- /dev/null +++ b/apps/api/tests/test_projection_service.py @@ -0,0 +1,47 @@ +"""Unit tests for the F23 projection-repository composition root (env-flagged). + +Hermetic (no Postgres): proves ``build_projection_repository`` defaults to the +in-memory store and swaps to the DB-backed repository only when +``FORGE_PROJECTION_BACKEND=db``. The ``db`` branch stubs the shared session +factory so no engine/connection is opened. +""" + +from __future__ import annotations + +import pytest + +import forge_api.db as db +from forge_api.services.projection_repository_db import SqlAlchemyProjectionRepository +from forge_api.services.projection_service import build_projection_repository +from forge_api.settings import get_settings +from forge_spec import InMemoryProjectionRepository + + +@pytest.fixture(autouse=True) +def _reset_settings() -> None: + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def test_defaults_to_in_memory(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FORGE_PROJECTION_BACKEND", raising=False) + get_settings.cache_clear() + assert isinstance(build_projection_repository(), InMemoryProjectionRepository) + + +def test_memory_flag_selects_in_memory(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FORGE_PROJECTION_BACKEND", "memory") + get_settings.cache_clear() + assert isinstance(build_projection_repository(), InMemoryProjectionRepository) + + +def test_db_flag_selects_sqlalchemy_repo(monkeypatch: pytest.MonkeyPatch) -> None: + sentinel = object() + monkeypatch.setattr(db, "get_session_factory", lambda: sentinel) + monkeypatch.setenv("FORGE_PROJECTION_BACKEND", "db") + get_settings.cache_clear() + + repo = build_projection_repository() + assert isinstance(repo, SqlAlchemyProjectionRepository) + assert repo._sf is sentinel From 084aa3197469fd2685fca47d298b250466835c62 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 15:58:31 +0200 Subject: [PATCH 08/12] feat(db/pm-link-repository): Postgres persistence Co-Authored-By: Claude Fable 5 --- .../forge_api/services/pm_link_repository.py | 39 ++ .../services/pm_link_repository_db.py | 227 +++++++++ apps/api/forge_api/settings.py | 9 + apps/api/tests/test_pm_link_repository_db.py | 441 ++++++++++++++++++ 4 files changed, 716 insertions(+) create mode 100644 apps/api/forge_api/services/pm_link_repository.py create mode 100644 apps/api/forge_api/services/pm_link_repository_db.py create mode 100644 apps/api/tests/test_pm_link_repository_db.py diff --git a/apps/api/forge_api/services/pm_link_repository.py b/apps/api/forge_api/services/pm_link_repository.py new file mode 100644 index 00000000..58b31d92 --- /dev/null +++ b/apps/api/forge_api/services/pm_link_repository.py @@ -0,0 +1,39 @@ +"""F18 composition root: the PM-sync link-repository seam. + +:func:`build_link_repository` selects the ``LinkRepository`` backend that a +:class:`~forge_integrations.pm.sync_engine.PMSyncEngine` links durably through, +via ``FORGE_PM_LINK_BACKEND`` (default ``memory``), exactly like the sibling +``FORGE_BOARD_BACKEND`` / ``FORGE_APPROVAL_BACKEND`` / ``FORGE_PROJECTION_BACKEND`` +seams: + +* ``memory`` (default) -> the hermetic + :class:`~forge_integrations.pm.sync_engine.InMemoryLinkRepository` (the + unit-test default; no Postgres, so every existing sync-engine test stays green + untouched); +* ``db`` -> the durable + :class:`~forge_api.services.pm_link_repository_db.DbLinkRepository` bound to the + shared session factory (mapping onto the ``pm_task_link`` table). + +Both satisfy the same ``LinkRepository`` protocol, so the swap is +behaviour-preserving: ``PMSyncEngine`` reads and writes through the port and +never learns which backend it got. The DB import is deferred so the default path +never imports ``forge_db`` / opens a connection. +""" + +from __future__ import annotations + +from forge_integrations.pm.sync_engine import InMemoryLinkRepository, LinkRepository + +__all__ = ["build_link_repository"] + + +def build_link_repository() -> LinkRepository: + """Return the link repository selected by ``FORGE_PM_LINK_BACKEND``.""" + from forge_api.settings import get_settings + + if get_settings().pm_link_backend == "db": + from forge_api.db import get_session_factory + from forge_api.services.pm_link_repository_db import DbLinkRepository + + return DbLinkRepository(get_session_factory()) + return InMemoryLinkRepository() diff --git a/apps/api/forge_api/services/pm_link_repository_db.py b/apps/api/forge_api/services/pm_link_repository_db.py new file mode 100644 index 00000000..bd8301ab --- /dev/null +++ b/apps/api/forge_api/services/pm_link_repository_db.py @@ -0,0 +1,227 @@ +"""Postgres-backed :class:`~forge_integrations.pm.sync_engine.LinkRepository` (F18). + +:class:`DbLinkRepository` is a drop-in, durable alternative to +:class:`~forge_integrations.pm.sync_engine.InMemoryLinkRepository` that satisfies +the **same** ``LinkRepository`` protocol (``get`` / ``get_by_forge_task`` / +``get_by_external`` / ``upsert`` / ``delete`` / ``list_by_state``) — so the F18 +composition root swaps it in behind ``FORGE_PM_LINK_BACKEND=db`` with no +behavioural change. The default stays ``memory`` and the in-memory store remains +the unit-test default (``PMSyncEngine`` is unit-tested against the in-memory +seam), so every existing sync-engine test stays green untouched. + +It lives in ``apps/api`` (not the ``forge_integrations`` SDK, which is +deliberately DB-free — the sync-engine module docstring says the engine "can be +wired to the real board service or to in-memory fakes"), exactly like the +sibling :class:`~forge_api.services.approval_repository_db.SqlAlchemyApprovalRepository`. +It maps the engine's :class:`~forge_integrations.pm.sync_engine.LinkRecord` onto +the canonical F18 ORM row (``forge_db.models.PMTaskLink`` / table +``pm_task_link``, migration ``0002``) — the very table the API-side +``PMConnectionService`` already reads, so a ``db``-backed engine and the API +service share one durable link store. + +Behaviour parity with the in-memory store is exact and intentional: + +* ``upsert`` is keyed by ``LinkRecord.id`` (the row primary key): an unknown id + inserts, a known id rewrites every field — the DB analogue of the in-memory + ``self._by_id[link.id] = link.model_copy(deep=True)``, and it returns the + stored record verbatim (a deep copy); +* every read maps the persisted row back to a fresh ``LinkRecord`` (never a live + ORM object), mirroring the in-memory store's ``model_copy(deep=True)`` so a + mutation of a returned record never leaks back into the store; +* ``get_by_forge_task`` / ``get_by_external`` scope by ``connection_id`` and read + as absent (``None``) for an unknown key; ``list_by_state`` filters by + ``(connection_id, sync_state)`` and returns rows in a deterministic order. + +The contracts enums the ``LinkRecord`` carries (``PMProvider`` / ``PMSyncState`` +from ``forge_contracts.pm``) are value-identical to the DB enums the row stores +(``forge_db.models.enums``), so the mapping is a straight ``Enum(value)`` on each +boundary. + +Storage-boundary divergences (shared with every DB-backed repo here; both still +satisfy the same protocol): + +* ``workspace_id`` / ``connection_id`` / ``forge_task_id`` are **real** foreign + keys, so an ``upsert`` requires those parent rows (``workspace`` / + ``pm_connection`` / ``task``) to exist; the engine only ever links tasks/ + connections it has just written, so this never surfaces in normal flow; +* the ``(connection_id, external_id)`` and ``(connection_id, forge_task_id)`` + unique constraints are enforced by the database — the engine always resolves an + existing link via ``get_by_forge_task`` / ``get_by_external`` before creating a + new ``LinkRecord``, so the wholesale upsert never trips them in normal flow, but + a direct duplicate insert (a *different* id colliding on one of those pairs) + raises at the boundary rather than silently succeeding. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from forge_contracts.pm import PMProvider as ContractsProvider +from forge_contracts.pm import PMSyncState as ContractsSyncState +from forge_db.models import PMTaskLink +from forge_db.models.enums import PMProvider as DbProvider +from forge_db.models.enums import PMSyncState as DbSyncState +from forge_integrations.pm.sync_engine import LinkRecord + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["DbLinkRepository"] + + +def _aware(value: datetime | None) -> datetime | None: + """Normalise a stored timestamp to timezone-aware UTC (SQLite reads naive).""" + if value is None: + return None + from datetime import UTC + + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +def _db_provider(value: ContractsProvider) -> DbProvider: + """Contracts ``PMProvider`` -> the DB ``PMProvider`` (value-identical enums).""" + return DbProvider(value.value) + + +def _db_state(value: ContractsSyncState) -> DbSyncState: + """Contracts ``PMSyncState`` -> the DB ``PMSyncState`` (value-identical enums).""" + return DbSyncState(value.value) + + +def _contracts_provider(value: object) -> ContractsProvider: + """DB ``PMProvider`` (or raw string) -> the contracts :class:`PMProvider`.""" + return ContractsProvider(value.value if isinstance(value, DbProvider) else str(value)) + + +def _contracts_state(value: object) -> ContractsSyncState: + """DB ``PMSyncState`` (or raw string) -> the contracts :class:`PMSyncState`.""" + return ContractsSyncState(value.value if isinstance(value, DbSyncState) else str(value)) + + +class DbLinkRepository: + """A Postgres-backed link repository (implements ``LinkRepository``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + # ------------------------------------------------------------------ # + # Mapping # + # ------------------------------------------------------------------ # + + def _apply(self, row: PMTaskLink, link: LinkRecord) -> None: + """Write every domain field onto ``row`` (used by ``upsert``).""" + row.workspace_id = link.workspace_id + row.connection_id = link.connection_id + row.forge_task_id = link.forge_task_id + row.provider = _db_provider(link.provider) + row.external_id = link.external_id + row.external_key = link.external_key + row.external_url = link.external_url + row.last_synced_at = link.last_synced_at + row.forge_version_at_sync = link.forge_version_at_sync + row.external_updated_at_at_sync = link.external_updated_at_at_sync + row.last_outbound_hash = link.last_outbound_hash + row.last_inbound_hash = link.last_inbound_hash + row.sync_state = _db_state(link.sync_state) + row.conflict_detail = ( + dict(link.conflict_detail) if link.conflict_detail is not None else None + ) + row.last_error = link.last_error + + def _to_domain(self, row: PMTaskLink) -> LinkRecord: + """Rebuild a fresh :class:`LinkRecord` from a persisted row.""" + return LinkRecord( + id=row.id, + connection_id=row.connection_id, + workspace_id=row.workspace_id, + forge_task_id=row.forge_task_id, + provider=_contracts_provider(row.provider), + external_id=row.external_id, + external_key=row.external_key, + external_url=row.external_url, + last_synced_at=_aware(row.last_synced_at), + forge_version_at_sync=row.forge_version_at_sync, + external_updated_at_at_sync=_aware(row.external_updated_at_at_sync), + last_outbound_hash=row.last_outbound_hash, + last_inbound_hash=row.last_inbound_hash, + sync_state=_contracts_state(row.sync_state), + conflict_detail=( + dict(row.conflict_detail) if row.conflict_detail is not None else None + ), + last_error=row.last_error, + ) + + # ------------------------------------------------------------------ # + # LinkRepository protocol # + # ------------------------------------------------------------------ # + + def get(self, link_id: UUID) -> LinkRecord | None: + with self._sf() as session: + row = session.get(PMTaskLink, link_id) + return self._to_domain(row) if row is not None else None + + def get_by_forge_task( + self, connection_id: UUID, forge_task_id: UUID + ) -> LinkRecord | None: + with self._sf() as session: + row = session.scalars( + select(PMTaskLink) + .where( + PMTaskLink.connection_id == connection_id, + PMTaskLink.forge_task_id == forge_task_id, + ) + .limit(1) + ).first() + return self._to_domain(row) if row is not None else None + + def get_by_external( + self, connection_id: UUID, external_id: str + ) -> LinkRecord | None: + with self._sf() as session: + row = session.scalars( + select(PMTaskLink) + .where( + PMTaskLink.connection_id == connection_id, + PMTaskLink.external_id == external_id, + ) + .limit(1) + ).first() + return self._to_domain(row) if row is not None else None + + def upsert(self, link: LinkRecord) -> LinkRecord: + with self._sf() as session: + row = session.get(PMTaskLink, link.id) + if row is None: + row = PMTaskLink(id=link.id) + self._apply(row, link) + session.add(row) + else: + self._apply(row, link) + session.commit() + return link.model_copy(deep=True) + + def delete(self, link_id: UUID) -> None: + with self._sf() as session: + row = session.get(PMTaskLink, link_id) + if row is not None: + session.delete(row) + session.commit() + + def list_by_state( + self, connection_id: UUID, state: ContractsSyncState + ) -> list[LinkRecord]: + with self._sf() as session: + rows = session.scalars( + select(PMTaskLink) + .where( + PMTaskLink.connection_id == connection_id, + PMTaskLink.sync_state == _db_state(state), + ) + .order_by(PMTaskLink.created_at.asc(), PMTaskLink.id.asc()) + ).all() + return [self._to_domain(r) for r in rows] diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index 6666b71c..7bc80f25 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -151,6 +151,15 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # ``FORGE_POLICY_AUDIT_BACKEND``. policy_audit_backend: str = "memory" + # F18 PM-sync link-repository backend selection. ``memory`` (default) keeps + # the hermetic, process-memory ``InMemoryLinkRepository`` (the sync-engine + # unit-test default, no Postgres); ``db`` wires the Postgres-backed + # ``DbLinkRepository`` behind the same ``LinkRepository`` protocol so a + # ``PMSyncEngine``'s Forge-task <-> external-issue links (and the loop- + # suppression hashes) land durably on the ``pm_task_link`` table. Read via + # ``FORGE_PM_LINK_BACKEND``. + pm_link_backend: str = "memory" + # F36 policy-override grant-store backend selection (J5). ``memory`` (default) # keeps the hermetic, process-memory ``InMemoryGrantStore`` (unit-test default, # no Postgres); ``db`` wires the Postgres-backed ``DbGrantStore`` behind the diff --git a/apps/api/tests/test_pm_link_repository_db.py b/apps/api/tests/test_pm_link_repository_db.py new file mode 100644 index 00000000..3d19697c --- /dev/null +++ b/apps/api/tests/test_pm_link_repository_db.py @@ -0,0 +1,441 @@ +"""Postgres integration tests for :class:`DbLinkRepository` (F18). + +Exercises the DB-backed PM-sync link repository against a real pgvector Postgres +via the shared ``pg_engine`` fixture (root ``conftest.py``): the +``LinkRepository`` protocol end-to-end — a full :class:`LinkRecord` round-trip +(every field, including the ``conflict_detail`` JSONB, the sync watermarks/hashes +and the timezone-aware ``last_synced_at`` / ``external_updated_at_at_sync``), +``upsert`` insert-vs-update keyed by id, connection-scoped +``get_by_forge_task`` / ``get_by_external``, ``list_by_state`` filtering + +connection scoping, ``delete`` (idempotent), the ``uq_pm_task_link_conn_task`` / +``uq_pm_task_link_conn_extid`` storage-boundary constraints, returned-copy +isolation, durability across repository instances, and structural conformance to +the same protocol the in-memory store implements. Skips cleanly (parked) when no +Postgres is reachable; runs under ``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in +the gate. + +Each behaviour mirrors the in-memory contract, so both backends are proven to +satisfy the same protocol identically. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.services.pm_link_repository_db import DbLinkRepository +from forge_contracts.pm import PMProvider as ContractsProvider +from forge_contracts.pm import PMSyncState as ContractsSyncState +from forge_db.base import Base +from forge_db.models import PMConnection, Project, Task, Workspace +from forge_db.models.enums import PMProvider as DbProvider +from forge_integrations.pm.sync_engine import InMemoryLinkRepository, LinkRecord + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + +_PROTOCOL_METHODS = ( + "get", + "get_by_forge_task", + "get_by_external", + "upsert", + "delete", + "list_by_state", +) + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def seed(factory: sessionmaker[Session]) -> dict[str, object]: + """A workspace + project + two PM connections + a pool of tasks (plus an + isolated second workspace), so link rows have real FK parents.""" + ws = uuid4() + other_ws = uuid4() + project_id = uuid4() + conn = uuid4() + conn2 = uuid4() + tasks = [uuid4() for _ in range(8)] + with factory() as session: + session.add(Workspace(id=ws, name="Acme", slug=f"acme-{uuid4().hex[:8]}")) + session.add( + Workspace(id=other_ws, name="Other", slug=f"other-{uuid4().hex[:8]}") + ) + session.flush() + session.add( + Project(id=project_id, workspace_id=ws, name="API", key=f"API{uuid4().hex[:4]}") + ) + session.flush() + session.add( + PMConnection( + id=conn, + workspace_id=ws, + provider=DbProvider.JIRA, + name="jira", + project_id=project_id, + external_project_key="ABC", + external_project_id="10000", + ) + ) + session.add( + PMConnection( + id=conn2, + workspace_id=ws, + provider=DbProvider.LINEAR, + name="linear", + project_id=project_id, + external_project_key="LIN", + external_project_id="lin-1", + ) + ) + session.flush() + for i, tid in enumerate(tasks): + session.add( + Task( + id=tid, + workspace_id=ws, + project_id=project_id, + key=f"API-{i}", + title=f"Task {i}", + ) + ) + session.commit() + return {"ws": ws, "other_ws": other_ws, "conn": conn, "conn2": conn2, "tasks": tasks} + + +@pytest.fixture +def repo(factory: sessionmaker[Session]) -> DbLinkRepository: + return DbLinkRepository(factory) + + +def _link( + seed: dict[str, object], + *, + task_index: int = 0, + connection: UUID | None = None, + external_id: str | None = None, + provider: ContractsProvider = ContractsProvider.jira, + **kw: object, +) -> LinkRecord: + tasks: list[UUID] = seed["tasks"] # type: ignore[assignment] + return LinkRecord( + connection_id=connection or seed["conn"], # type: ignore[arg-type] + workspace_id=seed["ws"], # type: ignore[arg-type] + forge_task_id=tasks[task_index], + provider=provider, + external_id=external_id or f"ext-{uuid4().hex[:8]}", + **kw, # type: ignore[arg-type] + ) + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_repo_conforms_to_link_repository_protocol(repo: DbLinkRepository) -> None: + mem = InMemoryLinkRepository() + for name in _PROTOCOL_METHODS: + assert callable(getattr(repo, name)) + assert callable(getattr(mem, name)) + + +# --------------------------------------------------------------------------- # +# upsert + get round-trip # +# --------------------------------------------------------------------------- # + + +def test_upsert_then_get_round_trips_every_field( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + last_synced = datetime(2026, 7, 5, 12, 0, 0, 123456, tzinfo=UTC) + ext_updated = datetime(2026, 7, 5, 11, 0, 0, 654321, tzinfo=UTC) + link = _link( + seed, + task_index=0, + external_id="EXT-1", + provider=ContractsProvider.jira, + external_key="ABC-1", + external_url="https://jira.example/ABC-1", + last_synced_at=last_synced, + forge_version_at_sync=3, + external_updated_at_at_sync=ext_updated, + last_outbound_hash="out-hash", + last_inbound_hash="in-hash", + sync_state=ContractsSyncState.conflict, + conflict_detail={"forge": {"v": 1}, "external": {"k": [1, 2, 3]}}, + last_error="boom", + ) + returned = repo.upsert(link) + # ``upsert`` returns the stored record verbatim (parity with the in-memory store). + assert returned.id == link.id + assert returned.external_id == "EXT-1" + + loaded = repo.get(link.id) + assert loaded is not None + assert loaded.id == link.id + assert loaded.connection_id == seed["conn"] + assert loaded.workspace_id == seed["ws"] + assert loaded.forge_task_id == seed["tasks"][0] # type: ignore[index] + assert loaded.provider is ContractsProvider.jira + assert loaded.external_id == "EXT-1" + assert loaded.external_key == "ABC-1" + assert loaded.external_url == "https://jira.example/ABC-1" + assert loaded.last_synced_at == last_synced + assert loaded.forge_version_at_sync == 3 + assert loaded.external_updated_at_at_sync == ext_updated + assert loaded.last_outbound_hash == "out-hash" + assert loaded.last_inbound_hash == "in-hash" + assert loaded.sync_state is ContractsSyncState.conflict + assert loaded.conflict_detail == {"forge": {"v": 1}, "external": {"k": [1, 2, 3]}} + assert loaded.last_error == "boom" + + +def test_upsert_defaults_round_trip( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + link = _link(seed, task_index=0, external_id="EXT-D") + repo.upsert(link) + loaded = repo.get(link.id) + assert loaded is not None + # ``LinkRecord`` string defaults are "" (NOT NULL columns), the rest None. + assert loaded.external_key == "" + assert loaded.external_url == "" + assert loaded.last_synced_at is None + assert loaded.forge_version_at_sync is None + assert loaded.external_updated_at_at_sync is None + assert loaded.last_outbound_hash is None + assert loaded.last_inbound_hash is None + assert loaded.conflict_detail is None + assert loaded.last_error is None + assert loaded.sync_state is ContractsSyncState.synced # LinkRecord default + + +def test_upsert_updates_existing_by_id( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + link = _link( + seed, task_index=0, external_id="EXT-1", sync_state=ContractsSyncState.pending_out + ) + repo.upsert(link) + + link.sync_state = ContractsSyncState.synced + link.last_outbound_hash = "new-hash" + link.external_key = "ABC-9" + repo.upsert(link) + + loaded = repo.get(link.id) + assert loaded is not None + assert loaded.sync_state is ContractsSyncState.synced + assert loaded.last_outbound_hash == "new-hash" + assert loaded.external_key == "ABC-9" + # An update keyed by id never inserts a second row. + assert repo.list_by_state(seed["conn"], ContractsSyncState.pending_out) == [] # type: ignore[arg-type] + assert len(repo.list_by_state(seed["conn"], ContractsSyncState.synced)) == 1 # type: ignore[arg-type] + + +def test_get_unknown_is_none(repo: DbLinkRepository) -> None: + assert repo.get(uuid4()) is None + + +def test_returned_records_are_isolated_copies( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + link = _link(seed, task_index=0, external_id="iso", conflict_detail={"k": 1}) + repo.upsert(link) + + loaded = repo.get(link.id) + assert loaded is not None + loaded.external_id = "MUTATED" + assert loaded.conflict_detail is not None + loaded.conflict_detail["k"] = 999 + + again = repo.get(link.id) + assert again is not None + assert again.external_id == "iso" + assert again.conflict_detail == {"k": 1} + + +# --------------------------------------------------------------------------- # +# get_by_forge_task / get_by_external: connection-scoped # +# --------------------------------------------------------------------------- # + + +def test_get_by_forge_task_scoped_by_connection( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + tasks: list[UUID] = seed["tasks"] # type: ignore[assignment] + link = _link(seed, task_index=0, connection=seed["conn"], external_id="E1") # type: ignore[arg-type] + repo.upsert(link) + + found = repo.get_by_forge_task(seed["conn"], tasks[0]) # type: ignore[arg-type] + assert found is not None and found.id == link.id + # Foreign connection / different task -> absent. + assert repo.get_by_forge_task(seed["conn2"], tasks[0]) is None # type: ignore[arg-type] + assert repo.get_by_forge_task(seed["conn"], tasks[1]) is None # type: ignore[arg-type] + + +def test_get_by_external_scoped_by_connection( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + link = _link(seed, task_index=0, connection=seed["conn"], external_id="E-777") # type: ignore[arg-type] + repo.upsert(link) + + found = repo.get_by_external(seed["conn"], "E-777") # type: ignore[arg-type] + assert found is not None and found.id == link.id + # Foreign connection / unknown external id -> absent. + assert repo.get_by_external(seed["conn2"], "E-777") is None # type: ignore[arg-type] + assert repo.get_by_external(seed["conn"], "missing") is None # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- # +# list_by_state: filtering + connection scope # +# --------------------------------------------------------------------------- # + + +def test_list_by_state_filters_and_scopes_by_connection( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + conn: UUID = seed["conn"] # type: ignore[assignment] + conn2: UUID = seed["conn2"] # type: ignore[assignment] + synced = ContractsSyncState.synced + l0 = _link(seed, task_index=0, connection=conn, external_id="A", sync_state=synced) + l1 = _link(seed, task_index=1, connection=conn, external_id="B", sync_state=synced) + l2 = _link(seed, task_index=2, connection=conn, external_id="C", sync_state=synced) + lc = _link( + seed, + task_index=3, + connection=conn, + external_id="D", + sync_state=ContractsSyncState.conflict, + ) + # A synced link on a *different* connection must never appear under ``conn``. + other = _link(seed, task_index=4, connection=conn2, external_id="E", sync_state=synced) + for link in (l0, l1, l2, lc, other): + repo.upsert(link) + + assert {r.id for r in repo.list_by_state(conn, ContractsSyncState.synced)} == { + l0.id, + l1.id, + l2.id, + } + assert [r.id for r in repo.list_by_state(conn, ContractsSyncState.conflict)] == [lc.id] + assert repo.list_by_state(conn, ContractsSyncState.error) == [] + assert [r.id for r in repo.list_by_state(conn2, ContractsSyncState.synced)] == [other.id] + + +# --------------------------------------------------------------------------- # +# delete # +# --------------------------------------------------------------------------- # + + +def test_delete_removes_link_and_is_idempotent( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + link = _link(seed, task_index=0, external_id="E-del") + repo.upsert(link) + assert repo.get(link.id) is not None + + repo.delete(link.id) + assert repo.get(link.id) is None + # Idempotent: deleting an already-removed / unknown id is a no-op. + repo.delete(link.id) + repo.delete(uuid4()) + + +# --------------------------------------------------------------------------- # +# Storage-boundary constraints + durability # +# --------------------------------------------------------------------------- # + + +def test_duplicate_forge_task_link_raises( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + """``uq_pm_task_link_conn_task`` blocks a second link for the same task.""" + repo.upsert(_link(seed, task_index=0, connection=seed["conn"], external_id="X1")) # type: ignore[arg-type] + with pytest.raises(IntegrityError): + repo.upsert(_link(seed, task_index=0, connection=seed["conn"], external_id="X2")) # type: ignore[arg-type] + + +def test_duplicate_external_id_link_raises( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + """``uq_pm_task_link_conn_extid`` blocks two tasks mapping to one external id.""" + repo.upsert(_link(seed, task_index=0, connection=seed["conn"], external_id="SAME")) # type: ignore[arg-type] + with pytest.raises(IntegrityError): + repo.upsert(_link(seed, task_index=1, connection=seed["conn"], external_id="SAME")) # type: ignore[arg-type] + + +def test_persists_across_repository_instances( + factory: sessionmaker[Session], seed: dict[str, object] +) -> None: + first = DbLinkRepository(factory) + link = _link(seed, task_index=0, external_id="dur") + first.upsert(link) + + second = DbLinkRepository(factory) + loaded = second.get(link.id) + assert loaded is not None and loaded.external_id == "dur" + + +# --------------------------------------------------------------------------- # +# Parity with the in-memory store (same protocol, identical behaviour) # +# --------------------------------------------------------------------------- # + + +def test_matches_in_memory_store_behaviour( + repo: DbLinkRepository, seed: dict[str, object] +) -> None: + conn: UUID = seed["conn"] # type: ignore[assignment] + tasks: list[UUID] = seed["tasks"] # type: ignore[assignment] + mem = InMemoryLinkRepository() + + synced = ContractsSyncState.synced + links = [ + _link(seed, task_index=0, connection=conn, external_id="p0", sync_state=synced), + _link( + seed, + task_index=1, + connection=conn, + external_id="p1", + sync_state=ContractsSyncState.conflict, + ), + _link(seed, task_index=2, connection=conn, external_id="p2", sync_state=synced), + ] + for link in links: + repo.upsert(link.model_copy(deep=True)) + mem.upsert(link.model_copy(deep=True)) + + # get agrees on both backends. + for link in links: + db_rec = repo.get(link.id) + mem_rec = mem.get(link.id) + assert db_rec is not None and mem_rec is not None + assert db_rec.external_id == mem_rec.external_id + assert db_rec.sync_state == mem_rec.sync_state + + # list_by_state agrees (order-independent; the protocol carries no ordering). + assert {r.id for r in repo.list_by_state(conn, ContractsSyncState.synced)} == { + r.id for r in mem.list_by_state(conn, ContractsSyncState.synced) + } + + # get_by_external + get_by_forge_task agree. + db_ext = repo.get_by_external(conn, "p1") + mem_ext = mem.get_by_external(conn, "p1") + assert db_ext is not None and mem_ext is not None and db_ext.id == mem_ext.id + + db_task = repo.get_by_forge_task(conn, tasks[0]) + mem_task = mem.get_by_forge_task(conn, tasks[0]) + assert db_task is not None and mem_task is not None + assert db_task.id == mem_task.id == links[0].id From 43857d7eaf844463f69e6e5e026de9aa7efd1a4c Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 17:04:32 +0200 Subject: [PATCH 09/12] feat(db/secret-vault-store): Postgres persistence Co-Authored-By: Claude Fable 5 --- apps/api/forge_api/auth/service.py | 27 +- apps/api/forge_api/auth/vault_db.py | 195 +++++++++ apps/api/forge_api/settings.py | 9 + apps/api/tests/test_secret_vault_store_db.py | 389 ++++++++++++++++++ packages/db/forge_db/models/__init__.py | 2 + packages/db/forge_db/models/secret.py | 117 ++++++ .../versions/0027_secret_vault_store.py | 71 ++++ packages/db/tests/test_models.py | 4 + 8 files changed, 811 insertions(+), 3 deletions(-) create mode 100644 apps/api/forge_api/auth/vault_db.py create mode 100644 apps/api/tests/test_secret_vault_store_db.py create mode 100644 packages/db/forge_db/models/secret.py create mode 100644 packages/db/migrations/versions/0027_secret_vault_store.py diff --git a/apps/api/forge_api/auth/service.py b/apps/api/forge_api/auth/service.py index 6cce02b5..d2451cbf 100644 --- a/apps/api/forge_api/auth/service.py +++ b/apps/api/forge_api/auth/service.py @@ -38,7 +38,7 @@ from forge_api.auth.oauth import OAuthClient, UnsupportedOAuthProviderError from forge_api.auth.providers import get_default_provider, resolve_secret from forge_api.auth.rbac import Permission, PermissionDeniedError, ensure -from forge_api.auth.vault import SecretVault +from forge_api.auth.vault import SecretStore, SecretVault from forge_api.deps import Principal from forge_api.observability.redaction import redact_text from forge_api.services.auth_audit import AuthAuditEmitter @@ -208,14 +208,35 @@ def _keyring_for_master(master: bytes) -> KeyRing: return KeyRing(keys, current_version) +def _build_secret_store() -> SecretStore | None: + """Return the secret-vault store selected by ``FORGE_SECRET_BACKEND``. + + ``memory`` (default) → ``None``, so :class:`SecretVault` falls back to the + hermetic :class:`~forge_api.auth.vault.InMemorySecretStore` (unit-test default, + no Postgres); ``db`` → the durable + :class:`~forge_api.auth.vault_db.DbSecretStore` bound to the shared session + factory. Both satisfy the same ``SecretStore`` seam, so the swap is + behaviour-preserving (add / get / list / remove + rotation's ``all_records``). + """ + from forge_api.settings import get_settings + + if get_settings().secret_backend == "db": + from forge_api.auth.vault_db import DbSecretStore + from forge_api.db import get_session_factory + + return DbSecretStore(get_session_factory()) + return None + + def _build_vault(master: bytes) -> SecretVault: """Construct the vault, selecting envelope vs single-tier cipher via config.""" cipher_subkey = _subkey(master, b"forge-cipher") + store = _build_secret_store() if _envelope_enabled(): keyring = _keyring_for_master(master) cipher = EnvelopeCipher(keyring, legacy=default_cipher(cipher_subkey)) - return SecretVault(cipher=cipher) - return SecretVault(cipher=default_cipher(cipher_subkey)) + return SecretVault(cipher=cipher, store=store) + return SecretVault(cipher=default_cipher(cipher_subkey), store=store) class AuthService: diff --git a/apps/api/forge_api/auth/vault_db.py b/apps/api/forge_api/auth/vault_db.py new file mode 100644 index 00000000..fc135e78 --- /dev/null +++ b/apps/api/forge_api/auth/vault_db.py @@ -0,0 +1,195 @@ +"""Postgres-backed encrypted secret store (secret-vault persistence). + +:class:`DbSecretStore` is a drop-in, durable alternative to +:class:`~forge_api.auth.vault.InMemorySecretStore` that satisfies the **same** +:class:`~forge_api.auth.vault.SecretStore` protocol (``add`` / ``get`` / ``list`` +/ ``remove``) the :class:`~forge_api.auth.vault.SecretVault` stores, reads, +lists, and rotates through. The composition root swaps it in behind +``FORGE_SECRET_BACKEND=db``; the default stays ``memory`` and the in-memory store +remains the unit-test default, so no existing behaviour changes. + +It maps the domain :class:`~forge_api.auth.vault.StoredSecret` onto the ``secret`` +ORM row one-to-one, preserving every field verbatim so a round-tripped record +equals the one the vault stored: + +* **Envelope encryption stays opaque.** ``ciphertext`` is persisted as the exact + bytes the cipher produced; the plaintext is never decrypted here, never logged, + and the ``__repr__`` on both the record and the row hides it. The vault's + read-time expiry (``SecretExpiredError``) and rotation (``rewrap_all`` via + ``all_records``) work unchanged because ``expires_at`` / ``key_version`` / + ``rotated_at`` round-trip faithfully. +* **Not-found / cross-tenant semantics.** ``get`` returns ``None`` (never raises) + for a missing *or* cross-workspace id, exactly like the in-memory store — so the + vault raises ``SecretNotFoundError`` at its boundary identically on both + backends. ``remove`` returns ``False`` for the same cases. +* **Domain-owned timestamps.** ``created_at`` / ``updated_at`` are persisted from + the record (not the DB clock), and the update path writes ``updated_at`` + explicitly so the column's ``onupdate`` never overrides a rewrap that left + ``updated_at`` untouched — the in-memory store mutates in place and keeps the + old value, and this reproduces that byte-for-byte. + +``add`` is an upsert (mirrors the in-memory ``dict[id] = record``): a fresh id +inserts, a repeated id updates in place, so ``rotate_secret`` / ``rewrap_all`` +(which re-``add`` a mutated record) persist correctly. ``all_records`` is the +cross-workspace listing the vault's ``rewrap_all`` / ``sweep_expired`` helpers +require of a non-in-memory store. +""" + +from __future__ import annotations + +import builtins +import uuid +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from sqlalchemy import delete, select, update + +from forge_api.auth.vault import StoredSecret +from forge_contracts.enums import APIKeyKind +from forge_db.models import Secret +from forge_db.models.enums import APIKeyKind as DbAPIKeyKind + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +__all__ = ["DbSecretStore"] + + +def _aware(value: datetime | None) -> datetime | None: + """Normalise a stored timestamp to timezone-aware UTC (defensive). + + A ``timestamptz`` reads back aware; a naive value (e.g. a SQLite round-trip) + is assumed UTC — matching the vault's own ``_is_expired`` tolerance so + read-time expiry stays correct on every dialect. + """ + if value is None: + return None + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +class DbSecretStore: + """A Postgres-backed encrypted secret store (implements ``SecretStore``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + # ------------------------------------------------------------------ # + # Mapping # + # ------------------------------------------------------------------ # + + def _mutable_values(self, record: StoredSecret) -> dict[str, Any]: + """Column kwargs for every field except the identity/created_at anchors.""" + return { + "workspace_id": record.workspace_id, + "name": record.name, + "kind": DbAPIKeyKind(record.kind.value), + "provider": record.provider, + "key_prefix": record.key_prefix, + "ciphertext": record.ciphertext, + "last_used_at": record.last_used_at, + "expires_at": record.expires_at, + "key_version": record.key_version, + "rotated_at": record.rotated_at, + } + + def _to_record(self, row: Secret) -> StoredSecret: + """Rebuild the exact :class:`StoredSecret` that produced ``row``.""" + return StoredSecret( + id=row.id, + workspace_id=row.workspace_id, + name=row.name, + kind=APIKeyKind(row.kind), + ciphertext=bytes(row.ciphertext), + provider=row.provider, + key_prefix=row.key_prefix, + created_at=_aware(row.created_at), # type: ignore[arg-type] + updated_at=_aware(row.updated_at), # type: ignore[arg-type] + last_used_at=_aware(row.last_used_at), + expires_at=_aware(row.expires_at), + key_version=row.key_version, + rotated_at=_aware(row.rotated_at), + ) + + # ------------------------------------------------------------------ # + # SecretStore protocol # + # ------------------------------------------------------------------ # + + def add(self, record: StoredSecret) -> None: + """Persist a record; upsert on a repeated id (mirrors the dict store). + + A repeated id is an in-place update (``rotate_secret`` / ``rewrap_all``); + ``updated_at`` is written explicitly so the column's ``onupdate`` never + overrides a rewrap that deliberately left it unchanged. + """ + with self._sf() as session: + exists = session.get(Secret, record.id) is not None + if exists: + session.execute( + update(Secret) + .where(Secret.id == record.id) + .values(updated_at=record.updated_at, **self._mutable_values(record)) + ) + else: + session.add( + Secret( + id=record.id, + created_at=record.created_at, + updated_at=record.updated_at, + secret_metadata={}, + **self._mutable_values(record), + ) + ) + session.commit() + + def get( + self, workspace_id: uuid.UUID, secret_id: uuid.UUID + ) -> StoredSecret | None: + """The record with ``secret_id`` in ``workspace_id``, else ``None``. + + Returns ``None`` (never raises) for a missing *or* cross-workspace id, so + the vault raises ``SecretNotFoundError`` identically to the in-memory store. + """ + with self._sf() as session: + row = session.get(Secret, secret_id) + if row is None or row.workspace_id != workspace_id: + return None + return self._to_record(row) + + def list(self, workspace_id: uuid.UUID) -> builtins.list[StoredSecret]: + """Every record in a workspace, oldest first (stable ordering).""" + with self._sf() as session: + rows = session.scalars( + select(Secret) + .where(Secret.workspace_id == workspace_id) + .order_by(Secret.created_at.asc(), Secret.id.asc()) + ).all() + return [self._to_record(r) for r in rows] + + def remove(self, workspace_id: uuid.UUID, secret_id: uuid.UUID) -> bool: + """Delete a workspace-scoped record; ``False`` if absent/cross-tenant.""" + with self._sf() as session: + result = session.execute( + delete(Secret).where( + Secret.id == secret_id, + Secret.workspace_id == workspace_id, + ) + ) + session.commit() + return bool(result.rowcount) + + # ------------------------------------------------------------------ # + # Rotation / sweep helper (not part of the protocol) # + # ------------------------------------------------------------------ # + + def all_records(self) -> builtins.list[StoredSecret]: + """Every record across all workspaces (vault rewrap/sweep helper). + + The vault's ``rewrap_all`` / ``sweep_expired`` span every tenant; a + non-in-memory store must expose this so KEK rotation and expiry hygiene + work identically to the in-memory ``_by_id`` iteration. + """ + with self._sf() as session: + rows = session.scalars( + select(Secret).order_by(Secret.created_at.asc(), Secret.id.asc()) + ).all() + return [self._to_record(r) for r in rows] diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index 7bc80f25..3d5d4c42 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -168,6 +168,15 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # are enforced by the database. Read via ``FORGE_OVERRIDE_GRANT_BACKEND``. override_grant_backend: str = "memory" + # Encrypted secret-vault store backend selection. ``memory`` (default) keeps + # the hermetic, process-memory ``InMemorySecretStore`` (unit-test default, no + # Postgres); ``db`` wires the Postgres-backed ``DbSecretStore`` behind the same + # ``SecretStore`` seam (``add`` / ``get`` / ``list`` / ``remove`` + rotation's + # ``all_records``) so envelope-encrypted BYOK secrets survive a restart. Only + # ever holds ciphertext — plaintext is never persisted. Read via + # ``FORGE_SECRET_BACKEND``. + secret_backend: str = "memory" + # Filesystem root for the spec engine's SDD artifacts (manifests, plans). spec_root: str = "specs" diff --git a/apps/api/tests/test_secret_vault_store_db.py b/apps/api/tests/test_secret_vault_store_db.py new file mode 100644 index 00000000..02df99c4 --- /dev/null +++ b/apps/api/tests/test_secret_vault_store_db.py @@ -0,0 +1,389 @@ +"""Postgres integration tests for :class:`DbSecretStore` (secret-vault persistence). + +Exercises the DB-backed encrypted secret store against a real pgvector Postgres +via the shared ``pg_engine`` fixture (root ``conftest.py``): the ``SecretStore`` +protocol end-to-end — round-trip of every :class:`StoredSecret` field, upsert +(rotation) semantics, per-workspace isolation (cross-tenant ``get``/``list``/ +``remove`` return nothing), oldest-first ordering, ``expires_at`` fidelity driving +the vault's read-time ``SecretExpiredError``, ``SecretNotFoundError`` for a missing +id, the workspace foreign-key constraint, durability across independent store +instances, ``all_records`` for cross-workspace rotation, and structural conformance +to the same frozen protocol the in-memory store implements. Skips cleanly (parked) +when no Postgres is reachable; runs under ``FORGE_TEST_DATABASE_URL`` (pgvector +:5433) in the gate. + +Security: the plaintext is never persisted and is never emitted by this module — +assertions compare booleans / ciphertext-absence, never printing a credential. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable, Iterator +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.auth.crypto import HmacAeadCipher, generate_key +from forge_api.auth.vault import ( + SecretExpiredError, + SecretNotFoundError, + SecretStore, + SecretVault, + StoredSecret, +) +from forge_api.auth.vault_db import DbSecretStore +from forge_contracts.enums import APIKeyKind +from forge_db.base import Base +from forge_db.models import Secret, Workspace + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + +# A non-secret ciphertext sentinel (this is opaque bytes, not a credential). +_CIPHERTEXT = b"\x02enveloped-opaque-blob-\x00\x01\x02" + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def make_workspace(factory: sessionmaker[Session]) -> Callable[[], uuid.UUID]: + """Insert a real workspace (the ``secret.workspace_id`` FK target) and return its id.""" + + def _make() -> uuid.UUID: + ws = Workspace(name="Acme", slug=f"acme-{uuid.uuid4().hex[:12]}") + with factory() as session: + session.add(ws) + session.commit() + return ws.id + + return _make + + +@pytest.fixture +def store(factory: sessionmaker[Session]) -> DbSecretStore: + return DbSecretStore(factory) + + +def _record(workspace_id: uuid.UUID, **kwargs: object) -> StoredSecret: + fields: dict[str, object] = { + "id": uuid.uuid4(), + "workspace_id": workspace_id, + "name": "anthropic", + "kind": APIKeyKind.MODEL_PROVIDER, + "ciphertext": _CIPHERTEXT, + } + fields.update(kwargs) + return StoredSecret(**fields) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_db_store_satisfies_secret_store_protocol(store: DbSecretStore) -> None: + assert isinstance(store, SecretStore) + + +# --------------------------------------------------------------------------- # +# Round-trip # +# --------------------------------------------------------------------------- # + + +def test_round_trip_preserves_all_fields( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + created = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + expires = created + timedelta(days=30) + used = created + timedelta(hours=1) + rotated = created + timedelta(days=2) + record = _record( + ws, + name="openai-prod", + kind=APIKeyKind.INTEGRATION_TOKEN, + provider="openai", + key_prefix="sk-o…", + created_at=created, + updated_at=created, + last_used_at=used, + expires_at=expires, + key_version=3, + rotated_at=rotated, + ) + store.add(record) + + loaded = store.get(ws, record.id) + assert loaded is not None + assert loaded.id == record.id + assert loaded.workspace_id == ws + assert loaded.name == "openai-prod" + assert loaded.kind is APIKeyKind.INTEGRATION_TOKEN + assert loaded.provider == "openai" + assert loaded.key_prefix == "sk-o…" + assert loaded.ciphertext == _CIPHERTEXT + assert loaded.created_at == created + assert loaded.last_used_at == used + assert loaded.expires_at == expires + assert loaded.key_version == 3 + assert loaded.rotated_at == rotated + + +def test_get_missing_returns_none( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + assert store.get(ws, uuid.uuid4()) is None + + +def test_plaintext_never_persisted( + store: DbSecretStore, + factory: sessionmaker[Session], + make_workspace: Callable[[], uuid.UUID], +) -> None: + """Only ciphertext is stored: a real plaintext never appears in the row bytes.""" + ws = make_workspace() + cipher = HmacAeadCipher(generate_key()) + plaintext = "sk-ant-PLAINTEXT-should-never-persist" + record = _record(ws, ciphertext=cipher.encrypt(plaintext)) + store.add(record) + + with factory() as session: + row = session.get(Secret, record.id) + assert row is not None + assert plaintext.encode() not in bytes(row.ciphertext) + # And the vault decrypts it back through the DB store. + vault = SecretVault(cipher=cipher, store=store) + assert vault.get_secret(ws, record.id) == plaintext + + +# --------------------------------------------------------------------------- # +# Upsert (rotation) semantics # +# --------------------------------------------------------------------------- # + + +def test_add_upserts_on_repeated_id( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + created = datetime(2026, 1, 1, tzinfo=UTC) + record = _record(ws, created_at=created, updated_at=created, key_version=1) + store.add(record) + + # Rotate in place: new ciphertext + bumped key_version + fresh updated_at, + # created_at unchanged (mirrors SecretVault.rotate_secret / rewrap_all). + rotated_at = datetime(2026, 2, 1, tzinfo=UTC) + record.ciphertext = b"\x02rewrapped-blob" + record.key_version = 2 + record.rotated_at = rotated_at + record.updated_at = rotated_at + store.add(record) + + loaded = store.get(ws, record.id) + assert loaded is not None + assert loaded.ciphertext == b"\x02rewrapped-blob" + assert loaded.key_version == 2 + assert loaded.rotated_at == rotated_at + assert loaded.created_at == created # anchor preserved + assert loaded.updated_at == rotated_at + assert len(store.list(ws)) == 1 # upsert, not insert + + +def test_upsert_keeps_updated_at_when_rewrap_leaves_it( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + """rewrap_all mutates ciphertext but not updated_at; the column's onupdate + must not silently bump it (parity with the in-place in-memory store).""" + ws = make_workspace() + stamp = datetime(2026, 3, 3, tzinfo=UTC) + record = _record(ws, created_at=stamp, updated_at=stamp) + store.add(record) + + record.ciphertext = b"\x02rewrapped-only" + record.key_version = 7 + # updated_at deliberately left as the original stamp. + store.add(record) + + loaded = store.get(ws, record.id) + assert loaded is not None + assert loaded.updated_at == stamp + assert loaded.key_version == 7 + + +# --------------------------------------------------------------------------- # +# Per-workspace isolation # +# --------------------------------------------------------------------------- # + + +def test_cross_tenant_get_returns_none( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws_a = make_workspace() + ws_b = make_workspace() + record = _record(ws_a) + store.add(record) + assert store.get(ws_a, record.id) is not None + assert store.get(ws_b, record.id) is None # same id, wrong tenant + + +def test_list_is_workspace_scoped_and_ordered( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws_a = make_workspace() + ws_b = make_workspace() + base = datetime(2026, 1, 1, tzinfo=UTC) + first = _record(ws_a, name="first", created_at=base, updated_at=base) + second = _record( + ws_a, name="second", created_at=base + timedelta(minutes=5), + updated_at=base + timedelta(minutes=5), + ) + other = _record(ws_b, name="other") + store.add(second) + store.add(first) + store.add(other) + + listed = store.list(ws_a) + assert [r.name for r in listed] == ["first", "second"] # oldest first + assert {r.workspace_id for r in listed} == {ws_a} + assert [r.name for r in store.list(ws_b)] == ["other"] + + +def test_remove_is_workspace_scoped( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws_a = make_workspace() + ws_b = make_workspace() + record = _record(ws_a) + store.add(record) + + assert store.remove(ws_b, record.id) is False # cross-tenant no-op + assert store.get(ws_a, record.id) is not None + assert store.remove(ws_a, record.id) is True + assert store.get(ws_a, record.id) is None + assert store.remove(ws_a, record.id) is False # already gone + + +# --------------------------------------------------------------------------- # +# Constraints + durability # +# --------------------------------------------------------------------------- # + + +def test_workspace_fk_is_enforced( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + """A secret for a non-existent workspace is rejected (schema-enforced scope).""" + ghost = uuid.uuid4() # never inserted as a workspace + with pytest.raises(IntegrityError): + store.add(_record(ghost)) + + +def test_all_records_spans_workspaces( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws_a = make_workspace() + ws_b = make_workspace() + store.add(_record(ws_a, name="a")) + store.add(_record(ws_b, name="b")) + names = {r.name for r in store.all_records()} + assert names == {"a", "b"} + + +def test_persists_across_store_instances( + factory: sessionmaker[Session], make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + record = _record(ws, name="durable") + DbSecretStore(factory).add(record) + + # A brand-new instance sees the durable row. + reloaded = DbSecretStore(factory).get(ws, record.id) + assert reloaded is not None + assert reloaded.name == "durable" + + +# --------------------------------------------------------------------------- # +# Vault semantics through the DB store (SecretNotFound / SecretExpired) # +# --------------------------------------------------------------------------- # + + +def test_vault_raises_not_found_through_db_store( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + vault = SecretVault(cipher=HmacAeadCipher(generate_key()), store=store) + with pytest.raises(SecretNotFoundError): + vault.get_secret(ws, uuid.uuid4()) + + +def test_vault_read_time_expiry_through_db_store( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + cipher = HmacAeadCipher(generate_key()) + vault = SecretVault(cipher=cipher, store=store) + past = datetime.now(UTC) - timedelta(days=1) + info = vault.put_secret( + workspace_id=ws, + name="expiring", + secret="sk-EXPIRING-value-000", + kind=APIKeyKind.MODEL_PROVIDER, + expires_at=past, + ) + with pytest.raises(SecretExpiredError): + vault.get_secret(ws, info.id) + # raw_record still yields the (encrypted) row for rotation. + assert vault.raw_record(ws, info.id).ciphertext # opaque bytes, non-empty + + +def test_vault_put_list_delete_round_trip_through_db_store( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + ws = make_workspace() + vault = SecretVault(cipher=HmacAeadCipher(generate_key()), store=store) + info = vault.put_secret( + workspace_id=ws, + name="listed", + secret="sk-LISTED-value-000", + kind=APIKeyKind.MCP_TOKEN, + provider="notion", + ) + listed = vault.list_secrets(ws) + assert [i.name for i in listed] == ["listed"] + assert listed[0].provider == "notion" + vault.delete_secret(ws, info.id) + assert vault.list_secrets(ws) == [] + with pytest.raises(SecretNotFoundError): + vault.delete_secret(ws, info.id) + + +# --------------------------------------------------------------------------- # +# Parity with the in-memory store (same protocol, identical behaviour) # +# --------------------------------------------------------------------------- # + + +def test_matches_in_memory_store_behaviour( + store: DbSecretStore, make_workspace: Callable[[], uuid.UUID] +) -> None: + from forge_api.auth.vault import InMemorySecretStore + + ws = make_workspace() + mem = InMemorySecretStore() + payloads = [_record(ws, name=f"k{i}") for i in range(3)] + for payload in payloads: + store.add(payload) + mem.add(payload) + + assert {r.id for r in store.list(ws)} == {r.id for r in mem.list(ws)} + target = payloads[1] + assert (store.get(ws, target.id) is None) == (mem.get(ws, target.id) is None) + assert store.remove(ws, target.id) == mem.remove(ws, target.id) + assert {r.id for r in store.list(ws)} == {r.id for r in mem.list(ws)} diff --git a/packages/db/forge_db/models/__init__.py b/packages/db/forge_db/models/__init__.py index c45f20d1..04827220 100644 --- a/packages/db/forge_db/models/__init__.py +++ b/packages/db/forge_db/models/__init__.py @@ -128,6 +128,7 @@ from forge_db.models.role_grant import RoleGrant from forge_db.models.runs import AgentRun, ApprovalRequest, SubAgentRun, WorkflowRun from forge_db.models.sandbox import SandboxInstance +from forge_db.models.secret import Secret from forge_db.models.sprint_velocity import ( SprintBurndownSnapshot, SprintScopeEvent, @@ -271,6 +272,7 @@ "ScimResourceType", "ScimToken", "ScopeType", + "Secret", "SkillProfile", "SpecDocument", "SpecStatus", diff --git a/packages/db/forge_db/models/secret.py b/packages/db/forge_db/models/secret.py new file mode 100644 index 00000000..fef389c3 --- /dev/null +++ b/packages/db/forge_db/models/secret.py @@ -0,0 +1,117 @@ +"""Durable backing table for the encrypted BYOK secret vault (secret-vault persist). + +The API's :class:`forge_api.auth.vault.InMemorySecretStore` is the storage +boundary behind :class:`~forge_api.auth.vault.SecretVault`: it holds +:class:`~forge_api.auth.vault.StoredSecret` records — the *envelope-encrypted* +credential (``ciphertext``), its :class:`~forge_contracts.enums.APIKeyKind`, the +HARD-13 envelope ``key_version`` the row's DEK is wrapped under, an optional +``expires_at`` (read-time expiry), and the owning ``workspace_id`` (per-workspace +isolation). This module is the Postgres backing for the *db* variant of that +store (``forge_api.auth.vault_db.DbSecretStore``): one row per stored secret. + +Why a **new** table rather than reusing ``api_key``: the two are genuinely +different concerns. ``api_key`` (F37 ``APIKey``) is the ORM row a *different* +BYOK code path already owns end-to-end; ``platform_api_key`` is inbound, +verify-only auth. This ``secret`` table is the durable image of the vault's +``StoredSecret`` boundary and nothing else, so the vault store can round-trip its +record verbatim without colliding with either existing table's invariants. + +Storage-boundary fidelity (load-bearing — the vault must round-trip exactly): + +* ``ciphertext`` is the opaque, envelope-encrypted blob (``LargeBinary``); the + plaintext is **never** persisted and ``__repr__`` never reveals the ciphertext. +* ``kind`` stores the :class:`APIKeyKind` value verbatim (VARCHAR + CHECK via the + shared :func:`enum_type`), byte-compatible with ``forge_contracts``. +* ``key_version`` / ``rotated_at`` mirror the HARD-13 envelope bookkeeping on + ``api_key`` (0023): the KEK version the DEK is wrapped under, and when it was + last re-wrapped — so KEK rotation (``rewrap_all``) can target + ``WHERE key_version < :current`` cheaply. +* ``created_at`` / ``updated_at`` are the domain record's own timestamps (the + repository persists them explicitly rather than letting the DB clock win), so a + round-tripped record equals the one the vault stored. +* ``secret_metadata`` is a reserved JSONB bag (defaults to ``{}``) for + forward-compatible per-secret annotations; the current ``StoredSecret`` carries + none, so it stays empty and never holds a credential. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import ( + DateTime, + Index, + LargeBinary, + SmallInteger, + String, + text, +) +from sqlalchemy.orm import Mapped, mapped_column + +from forge_db.base import WorkspaceScopedModel, enum_type, json_type +from forge_db.models.enums import APIKeyKind + + +class Secret(WorkspaceScopedModel): + """One persisted, envelope-encrypted BYOK secret (the vault's ``StoredSecret``). + + Tenant-scoped like every credential row (``workspace_id`` FK, CASCADE) so the + per-workspace isolation the vault enforces in code is also enforced by the + schema. The primary key is the domain record's own ``id`` (client-generated in + :meth:`SecretVault.put_secret`), preserved verbatim across the round-trip. + """ + + __tablename__ = "secret" + __table_args__ = ( + # Envelope-KEK rotation targets ``WHERE key_version < :current`` (0023 parity). + Index("ix_secret_key_version", "key_version"), + # Expiry sweep / read-time-expiry queries touch only rows that can expire. + Index( + "ix_secret_expires_at", + "expires_at", + postgresql_where=text("expires_at IS NOT NULL"), + ), + ) + + #: Human label for the credential (unique-per-workspace is a caller concern, + #: not enforced here — the in-memory store imposes no such constraint). + name: Mapped[str] = mapped_column(String(255), nullable=False) + #: BYOK key taxonomy; stored as the enum ``value`` (byte-compatible with contracts). + kind: Mapped[APIKeyKind] = mapped_column(enum_type(APIKeyKind), nullable=False) + #: Optional provider label (e.g. ``anthropic``); redacted view only. + provider: Mapped[str | None] = mapped_column(String(64), nullable=True) + #: Short display-safe prefix (e.g. ``sk-a…``); never the credential itself. + key_prefix: Mapped[str | None] = mapped_column(String(32), nullable=True) + #: The opaque envelope-encrypted credential blob (no plaintext, ever). + ciphertext: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) + #: Refreshed when the plaintext is decrypted for use (optional). + last_used_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + #: NULL = no expiry; a past value ⇒ the vault raises ``SecretExpiredError``. + expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + #: HARD-13 envelope bookkeeping: the KEK version the row's DEK is wrapped under. + key_version: Mapped[int] = mapped_column( + SmallInteger, default=1, server_default=text("1"), nullable=False + ) + #: When the DEK was last re-wrapped under a newer KEK (KEK-rotation audit). + rotated_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + #: Reserved forward-compat annotation bag (JSONB on Postgres); currently ``{}``. + secret_metadata: Mapped[dict[str, Any]] = mapped_column( + json_type(), default=dict, nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover - trivial, secret-safe + return ( + f"Secret(id={self.id!r}, workspace_id={self.workspace_id!r}, " + f"name={self.name!r}, kind={self.kind!r}, provider={self.provider!r}, " + f"key_prefix={self.key_prefix!r}, ciphertext=)" + ) + + +__all__ = ["Secret"] diff --git a/packages/db/migrations/versions/0027_secret_vault_store.py b/packages/db/migrations/versions/0027_secret_vault_store.py new file mode 100644 index 00000000..aef8927a --- /dev/null +++ b/packages/db/migrations/versions/0027_secret_vault_store.py @@ -0,0 +1,71 @@ +"""secret vault store: envelope-encrypted BYOK ``secret`` table. + +Backs the *db* variant of the API's secret vault store +(``forge_api.auth.vault`` — the storage boundary behind ``SecretVault``) with +real Postgres persistence. Creates one new, self-contained table: + +* ``secret`` — one row per stored :class:`~forge_api.auth.vault.StoredSecret`: + the envelope-encrypted ``ciphertext`` (no plaintext, ever), its + :class:`~forge_contracts.enums.APIKeyKind`, per-workspace scope + (``workspace_id`` FK, CASCADE), the HARD-13 envelope ``key_version`` / + ``rotated_at`` (KEK-rotation bookkeeping, mirroring 0023's ``api_key`` columns), + ``expires_at`` (read-time expiry), ``last_used_at``, display-safe ``key_prefix``, + ``created_at`` / ``updated_at`` (the domain record's own timestamps), and a + reserved ``secret_metadata`` JSONB bag. + +Distinct from ``api_key`` (F37's BYOK row, a different code path) and +``platform_api_key`` (inbound verify-only auth), so this revision only *adds* a +table and touches nothing existing. + +Foundation note (mirrors 0024/0025): ``forge_db``'s metadata is the source of +truth, so a fresh chain already provisions this table from the model. To stay +idiomatic *and* own an explicit, reversible step this migration is idempotent: +``upgrade`` creates only what is missing, ``downgrade`` drops only what this +revision introduced. Applies cleanly on SQLite (unit path) and pgvector Postgres. + +Revision ID: 0027_secret_vault_store +Revises: 0026_approval_repository_columns +Create Date: 2026-07-05 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) +from forge_db.base import Base + +# revision identifiers, used by Alembic. +revision: str = "0027_secret_vault_store" +down_revision: str | None = "0026_approval_repository_columns" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +# Tables owned by this revision (downgrade drops them in reverse). +_TABLES: tuple[str, ...] = ("secret",) + + +def _owned_tables() -> list[sa.Table]: + by_name = {t.name: t for t in Base.metadata.sorted_tables} + return [by_name[name] for name in _TABLES if name in by_name] + + +def _existing_tables() -> set[str]: + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def upgrade() -> None: + existing = _existing_tables() + to_create = [t for t in _owned_tables() if t.name not in existing] + if to_create: + Base.metadata.create_all(bind=op.get_bind(), tables=to_create) + + +def downgrade() -> None: + existing = _existing_tables() + to_drop = [t for t in reversed(_owned_tables()) if t.name in existing] + if to_drop: + Base.metadata.drop_all(bind=op.get_bind(), tables=to_drop) diff --git a/packages/db/tests/test_models.py b/packages/db/tests/test_models.py index b95843c2..4e808e7f 100644 --- a/packages/db/tests/test_models.py +++ b/packages/db/tests/test_models.py @@ -140,6 +140,10 @@ # optional workspace tag is a free UUID (``workspace_ref``), never the FK. "ObservabilityAuditEntry", "ObservabilityAuditChainHead", + # secret-vault persistence: durable backing for the encrypted BYOK secret + # store (the storage boundary behind ``SecretVault``). Workspace-scoped; + # holds only ciphertext, never plaintext. + "Secret", ] # Tables that are NOT the tenant root and therefore must carry a workspace FK. From d94202e9bfb18a4307fd3b2021cd186997b6f3eb Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 18:03:25 +0200 Subject: [PATCH 10/12] feat(db/idempotency-store): Postgres persistence Co-Authored-By: Claude Fable 5 --- apps/api/forge_api/middleware/__init__.py | 4 +- apps/api/forge_api/middleware/idempotency.py | 3 +- .../forge_api/middleware/idempotency_db.py | 173 +++++++++++ apps/api/forge_api/settings.py | 7 + apps/api/tests/test_idempotency_store_db.py | 278 ++++++++++++++++++ packages/db/forge_db/models/__init__.py | 2 + packages/db/forge_db/models/idempotency.py | 86 ++++++ .../versions/0028_idempotency_store.py | 71 +++++ packages/db/tests/test_migration.py | 36 +++ packages/db/tests/test_models.py | 8 + 10 files changed, 666 insertions(+), 2 deletions(-) create mode 100644 apps/api/forge_api/middleware/idempotency_db.py create mode 100644 apps/api/tests/test_idempotency_store_db.py create mode 100644 packages/db/forge_db/models/idempotency.py create mode 100644 packages/db/migrations/versions/0028_idempotency_store.py diff --git a/apps/api/forge_api/middleware/__init__.py b/apps/api/forge_api/middleware/__init__.py index fbc0d07e..0222e212 100644 --- a/apps/api/forge_api/middleware/__init__.py +++ b/apps/api/forge_api/middleware/__init__.py @@ -63,9 +63,11 @@ def install_middleware(app: FastAPI, settings: Settings) -> None: app.add_middleware(InFlightMiddleware) if settings.idempotency_enabled: + from forge_api.middleware.idempotency_db import build_idempotency_store + app.add_middleware( IdempotencyMiddleware, - store=InMemoryIdempotencyStore(), + store=build_idempotency_store(settings), ttl_s=settings.idempotency_ttl_seconds, enabled=True, ) diff --git a/apps/api/forge_api/middleware/idempotency.py b/apps/api/forge_api/middleware/idempotency.py index 8c7716e1..7623fde8 100644 --- a/apps/api/forge_api/middleware/idempotency.py +++ b/apps/api/forge_api/middleware/idempotency.py @@ -21,7 +21,7 @@ import threading import time from datetime import UTC, datetime -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Protocol, runtime_checkable from pydantic import BaseModel @@ -49,6 +49,7 @@ class StoredResponse(BaseModel): created_at: datetime +@runtime_checkable class IdempotencyStore(Protocol): """A tenant-scoped idempotency-key → response store.""" diff --git a/apps/api/forge_api/middleware/idempotency_db.py b/apps/api/forge_api/middleware/idempotency_db.py new file mode 100644 index 00000000..fb10efb9 --- /dev/null +++ b/apps/api/forge_api/middleware/idempotency_db.py @@ -0,0 +1,173 @@ +"""Postgres-backed HTTP idempotency store (idempotency-store persistence). + +:class:`DbIdempotencyStore` is a drop-in, durable alternative to +:class:`~forge_api.middleware.idempotency.InMemoryIdempotencyStore` that satisfies +the **same** :class:`~forge_api.middleware.idempotency.IdempotencyStore` protocol +(``get`` / ``put_if_absent``) that :class:`~forge_api.middleware.idempotency.IdempotencyMiddleware` +reserves and replays through. The composition root swaps it in behind +``FORGE_IDEMPOTENCY_BACKEND=db``; the default stays ``memory`` and the in-memory +store remains the unit-test default, so no existing behaviour changes. + +Behaviour parity with the in-memory store is exact and *intentional*: + +* **TTL / expiry semantics.** The in-memory store evicts on a monotonic clock; + this uses a persisted wall-clock ``expires_at`` (``now + max(1, ttl_s)``). A + read past ``expires_at`` returns ``None`` (the entry is *absent*), and a reserve + against an expired key overwrites it and returns ``True`` — mirroring the + in-memory ``_evict``-then-check exactly. +* **Concurrent reserve is atomic.** ``put_if_absent`` is a single + ``INSERT ... ON CONFLICT (key) DO UPDATE ... WHERE expires_at <= now``: a fresh + key inserts, an *expired* key is overwritten, and a *live* key is left untouched. + ``RETURNING`` tells the caller which happened — a row means we wrote (``True``), + no row means a live entry already existed (``False``). Two racing first-sights of + the same key thus collapse to exactly one winner at the database, so the guarded + side effect is cached once even across processes. +* **Byte-exact response round-trip.** The ``StoredResponse`` is stored as its JSONB + image with the ``body`` base64-encoded (JSON holds no raw bytes); ``get`` decodes + it back verbatim, and ``created_at`` is persisted from the record (not the DB + clock), so a replayed response equals the original the middleware captured. +""" + +from __future__ import annotations + +import base64 +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any + +from sqlalchemy import delete, select +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from forge_api.middleware.idempotency import ( + IdempotencyStore, + InMemoryIdempotencyStore, + StoredResponse, +) +from forge_db.models import IdempotencyKey + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + + from forge_api.settings import Settings + +__all__ = ["DbIdempotencyStore", "build_idempotency_store"] + + +def _aware(value: datetime) -> datetime: + """Normalise a stored timestamp to timezone-aware UTC (defensive). + + A ``timestamptz`` reads back aware; a naive value (e.g. a SQLite round-trip) + is assumed UTC so the expiry comparison and the replayed ``created_at`` stay + correct on every dialect. + """ + return value if value.tzinfo is not None else value.replace(tzinfo=UTC) + + +def _dump(value: StoredResponse) -> dict[str, Any]: + """The JSONB image of a ``StoredResponse`` (body base64-encoded).""" + return { + "request_hash": value.request_hash, + "status_code": value.status_code, + "content_type": value.content_type, + "body_b64": base64.b64encode(value.body).decode("ascii"), + } + + +def _load(payload: dict[str, Any], created_at: datetime) -> StoredResponse: + """Rebuild the exact ``StoredResponse`` that produced ``payload``.""" + return StoredResponse( + request_hash=payload["request_hash"], + status_code=payload["status_code"], + content_type=payload.get("content_type", "application/json"), + body=base64.b64decode(payload["body_b64"]), + created_at=_aware(created_at), + ) + + +class DbIdempotencyStore: + """A Postgres-backed idempotency store (implements ``IdempotencyStore``).""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + self._sf = session_factory + + def get(self, key: str) -> StoredResponse | None: + """The cached response for ``key`` if present *and* unexpired, else ``None``.""" + now = datetime.now(UTC) + with self._sf() as session: + row = session.execute( + select(IdempotencyKey).where(IdempotencyKey.key == key) + ).scalar_one_or_none() + if row is None or _aware(row.expires_at) <= now: + return None + return _load(row.response, row.created_at) + + def put_if_absent(self, key: str, value: StoredResponse, ttl_s: int) -> bool: + """Reserve ``key`` iff no *live* entry exists; return ``True`` when it wrote. + + Atomic get-or-set: a fresh key inserts, an expired key is overwritten, a + live key is a no-op. ``RETURNING`` distinguishes the write from the no-op. + """ + now = datetime.now(UTC) + expires_at = now + timedelta(seconds=max(1, ttl_s)) + payload = _dump(value) + stmt = ( + pg_insert(IdempotencyKey) + .values( + key=key, + response=payload, + created_at=value.created_at, + expires_at=expires_at, + ) + .on_conflict_do_update( + index_elements=["key"], + set_={ + "response": payload, + "created_at": value.created_at, + "expires_at": expires_at, + }, + where=IdempotencyKey.expires_at <= now, + ) + .returning(IdempotencyKey.key) + ) + with self._sf() as session: + wrote = session.execute(stmt).scalar_one_or_none() is not None + session.commit() + return wrote + + def purge_expired(self) -> int: + """Delete every entry past its TTL; return the count (sweep helper). + + Not part of the ``IdempotencyStore`` protocol — an operator-facing hygiene + path (the in-memory store evicts lazily on access; a durable table needs an + explicit sweep). ``get`` / ``put_if_absent`` already treat an expired row as + absent, so this only reclaims space. + """ + now = datetime.now(UTC) + with self._sf() as session: + result = session.execute( + delete(IdempotencyKey).where(IdempotencyKey.expires_at <= now) + ) + session.commit() + return int(result.rowcount or 0) + + +# --------------------------------------------------------------------------- # +# Composition root # +# --------------------------------------------------------------------------- # + + +def build_idempotency_store(settings: Settings | None = None) -> IdempotencyStore: + """Return the idempotency store selected by ``FORGE_IDEMPOTENCY_BACKEND``. + + ``memory`` (default) → the hermetic :class:`InMemoryIdempotencyStore` (unit-test + default, no Postgres); ``db`` → the durable :class:`DbIdempotencyStore` bound to + the shared session factory. Both satisfy the same frozen ``IdempotencyStore`` + protocol, so the middleware behaves identically on either. + """ + from forge_api.settings import get_settings + + settings = settings or get_settings() + if settings.idempotency_backend == "db": + from forge_api.db import get_session_factory + + return DbIdempotencyStore(get_session_factory()) + return InMemoryIdempotencyStore() diff --git a/apps/api/forge_api/settings.py b/apps/api/forge_api/settings.py index 3d5d4c42..cdc840bc 100644 --- a/apps/api/forge_api/settings.py +++ b/apps/api/forge_api/settings.py @@ -257,6 +257,13 @@ def _apply_legacy_aliases(cls, data: Any) -> Any: # request carries no key. idempotency_enabled: bool = True idempotency_ttl_seconds: int = 86_400 + # HTTP idempotency-store backend selection. ``memory`` (default) keeps the + # hermetic, process-memory ``InMemoryIdempotencyStore`` (unit-test default, no + # Postgres); ``db`` wires the Postgres-backed ``DbIdempotencyStore`` behind the + # same ``IdempotencyStore`` protocol so the cached idempotency-key responses + # survive a restart and dedup across processes. Read via + # ``FORGE_IDEMPOTENCY_BACKEND``. + idempotency_backend: str = "memory" # Graceful-shutdown request drain grace: on SIGTERM readiness flips to 503 and # the app waits up to this long for in-flight requests before tearing down. shutdown_drain_seconds: int = 30 diff --git a/apps/api/tests/test_idempotency_store_db.py b/apps/api/tests/test_idempotency_store_db.py new file mode 100644 index 00000000..94fe6cc4 --- /dev/null +++ b/apps/api/tests/test_idempotency_store_db.py @@ -0,0 +1,278 @@ +"""Postgres integration tests for :class:`DbIdempotencyStore` (idempotency-store persistence). + +Exercises the DB-backed HTTP idempotency store against a real pgvector Postgres +via the shared ``pg_engine`` fixture (root ``conftest.py``): the +``IdempotencyStore`` protocol end-to-end — byte-exact round-trip of every +``StoredResponse`` field, the ``put_if_absent`` reserve semantics (first write +wins, a live key is a no-op, an *expired* key is overwritten), read-time expiry, +durability across independent store instances, the ``key`` UNIQUE constraint, the +``purge_expired`` sweep, a full ``IdempotencyMiddleware`` round-trip (retry +replays, side effect runs once) driven through the DB store, and structural +conformance to the same frozen protocol the in-memory store implements. Skips +cleanly (parked) when no Postgres is reachable; runs under +``FORGE_TEST_DATABASE_URL`` (pgvector :5433) in the gate. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +import pytest +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.middleware.idempotency import ( + IdempotencyMiddleware, + IdempotencyStore, + InMemoryIdempotencyStore, + StoredResponse, +) +from forge_api.middleware.idempotency_db import DbIdempotencyStore +from forge_db.base import Base +from forge_db.models import IdempotencyKey + +pytestmark = [pytest.mark.postgres, pytest.mark.usefixtures("pg_engine")] + + +@pytest.fixture +def factory(pg_engine) -> Iterator[sessionmaker[Session]]: + Base.metadata.create_all(pg_engine) + try: + yield sessionmaker(bind=pg_engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(pg_engine) + + +@pytest.fixture +def store(factory: sessionmaker[Session]) -> DbIdempotencyStore: + return DbIdempotencyStore(factory) + + +def _response(**kwargs: object) -> StoredResponse: + fields: dict[str, object] = { + "request_hash": "reqhash-0001", + "status_code": 200, + "body": b'{"ok":true}', + "content_type": "application/json", + "created_at": datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC), + } + fields.update(kwargs) + return StoredResponse(**fields) # type: ignore[arg-type] + + +def _insert_expired( + factory: sessionmaker[Session], key: str, value: StoredResponse +) -> None: + """Insert a row whose TTL is already in the past (a stale cache entry).""" + import base64 + + with factory() as session: + session.add( + IdempotencyKey( + key=key, + response={ + "request_hash": value.request_hash, + "status_code": value.status_code, + "content_type": value.content_type, + "body_b64": base64.b64encode(value.body).decode("ascii"), + }, + created_at=value.created_at, + expires_at=datetime.now(UTC) - timedelta(hours=1), + ) + ) + session.commit() + + +# --------------------------------------------------------------------------- # +# Protocol conformance # +# --------------------------------------------------------------------------- # + + +def test_db_store_satisfies_idempotency_store_protocol(store: DbIdempotencyStore) -> None: + assert isinstance(store, IdempotencyStore) + + +# --------------------------------------------------------------------------- # +# Round-trip # +# --------------------------------------------------------------------------- # + + +def test_round_trip_preserves_all_fields(store: DbIdempotencyStore) -> None: + created = datetime(2026, 6, 7, 8, 9, 10, 123456, tzinfo=UTC) + value = _response( + request_hash="abc123", + status_code=201, + body=b'{"id":42,"name":"x"}', + content_type="application/json; charset=utf-8", + created_at=created, + ) + assert store.put_if_absent("k-round", value, 3600) is True + + loaded = store.get("k-round") + assert loaded is not None + assert loaded.request_hash == "abc123" + assert loaded.status_code == 201 + assert loaded.body == b'{"id":42,"name":"x"}' + assert loaded.content_type == "application/json; charset=utf-8" + assert loaded.created_at == created + + +def test_round_trip_preserves_non_utf8_body(store: DbIdempotencyStore) -> None: + """The body is opaque bytes (base64 in JSONB): arbitrary bytes round-trip exactly.""" + raw = bytes(range(256)) # includes non-UTF-8 bytes + assert store.put_if_absent("k-bin", _response(body=raw), 3600) is True + loaded = store.get("k-bin") + assert loaded is not None + assert loaded.body == raw + + +def test_get_missing_returns_none(store: DbIdempotencyStore) -> None: + assert store.get("nope") is None + + +# --------------------------------------------------------------------------- # +# put_if_absent reserve semantics # +# --------------------------------------------------------------------------- # + + +def test_put_if_absent_first_write_wins(store: DbIdempotencyStore) -> None: + first = _response(request_hash="first", body=b"one") + second = _response(request_hash="second", body=b"two") + assert store.put_if_absent("k", first, 3600) is True + assert store.put_if_absent("k", second, 3600) is False # live entry: no-op + + loaded = store.get("k") + assert loaded is not None + assert loaded.request_hash == "first" # original preserved, not overwritten + assert loaded.body == b"one" + + +def test_put_if_absent_overwrites_expired_entry( + store: DbIdempotencyStore, factory: sessionmaker[Session] +) -> None: + stale = _response(request_hash="stale", body=b"old") + _insert_expired(factory, "k-exp", stale) + + fresh = _response(request_hash="fresh", body=b"new") + assert store.put_if_absent("k-exp", fresh, 3600) is True # expired ⇒ overwrite + + loaded = store.get("k-exp") + assert loaded is not None + assert loaded.request_hash == "fresh" + assert loaded.body == b"new" + + +# --------------------------------------------------------------------------- # +# Expiry # +# --------------------------------------------------------------------------- # + + +def test_get_treats_expired_as_absent( + store: DbIdempotencyStore, factory: sessionmaker[Session] +) -> None: + _insert_expired(factory, "k-old", _response()) + assert store.get("k-old") is None + + +def test_purge_expired_removes_only_stale_rows( + store: DbIdempotencyStore, factory: sessionmaker[Session] +) -> None: + _insert_expired(factory, "stale-1", _response()) + _insert_expired(factory, "stale-2", _response()) + assert store.put_if_absent("live", _response(), 3600) is True + + assert store.purge_expired() == 2 + with factory() as session: + remaining = {row.key for row in session.query(IdempotencyKey).all()} + assert remaining == {"live"} + + +# --------------------------------------------------------------------------- # +# Constraints + durability # +# --------------------------------------------------------------------------- # + + +def test_key_unique_constraint_enforced(factory: sessionmaker[Session]) -> None: + """The ``key`` UNIQUE index (the ON CONFLICT target) is schema-enforced.""" + row = IdempotencyKey( + key="dupe", + response={"request_hash": "h", "status_code": 200, "content_type": "x", "body_b64": ""}, + expires_at=datetime.now(UTC) + timedelta(hours=1), + ) + dupe = IdempotencyKey( + key="dupe", + response={"request_hash": "h", "status_code": 200, "content_type": "x", "body_b64": ""}, + expires_at=datetime.now(UTC) + timedelta(hours=1), + ) + with pytest.raises(IntegrityError), factory() as session: + session.add_all([row, dupe]) + session.commit() + + +def test_persists_across_store_instances(factory: sessionmaker[Session]) -> None: + value = _response(request_hash="durable", body=b"kept") + assert DbIdempotencyStore(factory).put_if_absent("k-dur", value, 3600) is True + + reloaded = DbIdempotencyStore(factory).get("k-dur") + assert reloaded is not None + assert reloaded.request_hash == "durable" + assert reloaded.body == b"kept" + + +# --------------------------------------------------------------------------- # +# Full middleware round-trip through the DB store # +# --------------------------------------------------------------------------- # + + +def test_middleware_replays_through_db_store(factory: sessionmaker[Session]) -> None: + counter = {"n": 0} + app = FastAPI() + + @app.post("/work") + async def work(request: Request) -> dict[str, int]: + counter["n"] += 1 + body = await request.json() + return {"run": counter["n"], "echo": body.get("v", 0)} + + app.add_middleware( + IdempotencyMiddleware, store=DbIdempotencyStore(factory), ttl_s=3600 + ) + client = TestClient(app) + headers = {"Idempotency-Key": uuid.uuid4().hex} + + first = client.post("/work", json={"v": 1}, headers=headers) + second = client.post("/work", json={"v": 1}, headers=headers) + assert first.status_code == second.status_code == 200 + assert first.json() == second.json() # identical replayed body + assert counter["n"] == 1 # side effect ran exactly once (durable dedup) + assert second.headers.get("idempotency-replayed") == "true" + + # A different body under the same key is a client bug → 422, still no re-run. + mismatch = client.post("/work", json={"v": 999}, headers=headers) + assert mismatch.status_code == 422 + assert counter["n"] == 1 + + +# --------------------------------------------------------------------------- # +# Parity with the in-memory store (same protocol, identical behaviour) # +# --------------------------------------------------------------------------- # + + +def test_matches_in_memory_store_behaviour(store: DbIdempotencyStore) -> None: + mem = InMemoryIdempotencyStore() + value = _response(request_hash="parity", body=b"same") + + assert store.put_if_absent("k", value, 3600) == mem.put_if_absent("k", value, 3600) + assert store.put_if_absent("k", value, 3600) == mem.put_if_absent("k", value, 3600) + assert (store.get("k") is None) == (mem.get("k") is None) + assert (store.get("missing") is None) == (mem.get("missing") is None) + + db_hit = store.get("k") + mem_hit = mem.get("k") + assert db_hit is not None and mem_hit is not None + assert db_hit.request_hash == mem_hit.request_hash + assert db_hit.body == mem_hit.body diff --git a/packages/db/forge_db/models/__init__.py b/packages/db/forge_db/models/__init__.py index 04827220..46362731 100644 --- a/packages/db/forge_db/models/__init__.py +++ b/packages/db/forge_db/models/__init__.py @@ -80,6 +80,7 @@ UserRole, WorkflowState, ) +from forge_db.models.idempotency import IdempotencyKey from forge_db.models.incidents import ( IncidentAlert, IncidentEvent, @@ -204,6 +205,7 @@ "GateCheckName", "GateCheckStatus", "HealthStatus", + "IdempotencyKey", "Incident", "IncidentAlert", "IncidentEvent", diff --git a/packages/db/forge_db/models/idempotency.py b/packages/db/forge_db/models/idempotency.py new file mode 100644 index 00000000..c7d5fc22 --- /dev/null +++ b/packages/db/forge_db/models/idempotency.py @@ -0,0 +1,86 @@ +"""Durable backing table for the HTTP idempotency-key response cache (idempotency-store persist). + +The API's :class:`forge_api.middleware.idempotency.InMemoryIdempotencyStore` is +the store behind :class:`~forge_api.middleware.idempotency.IdempotencyMiddleware`: +a tenant-scoped ``key`` → :class:`~forge_api.middleware.idempotency.StoredResponse` +map with a per-entry TTL, so a retried unsafe request carrying the same +``Idempotency-Key`` replays the first response instead of re-running the side +effect. This module is the Postgres backing for the *db* variant of that store +(:class:`forge_api.middleware.idempotency_db.DbIdempotencyStore`): one row per +cached response. + +Why a **new** table rather than reusing a per-domain ``idempotency_key`` column: +the two are unrelated concerns. ``automation_execution.idempotency_key`` and +``deployment.idempotency_key`` dedupe a *specific domain command* at the service +layer; this is the *transport-level* HTTP response cache the middleware owns, +holding an opaque cached response for any unsafe route. There is no general HTTP +idempotency store table, so this ``idempotency_key`` table is it. + +Storage-boundary fidelity (the middleware must round-trip a ``StoredResponse`` +exactly, or a replay would differ from the original): + +* ``key`` is the middleware's fully-qualified, tenant-scoped cache key + (``forge:idem::
``); it is a UNIQUE column (not the PK) because + the foundation mandates a surrogate UUID ``id`` PK + timestamps on every table + (same deviation :class:`~forge_db.models.sso.SamlReplay` documents). The UNIQUE + index is the ``ON CONFLICT`` target the reserve/get-or-set path serializes on. +* ``response`` is the JSONB image of the ``StoredResponse`` value object: + ``request_hash`` (the middleware's 422-on-mismatch guard), ``status_code``, + ``content_type``, and the response ``body`` as a base64 ``body_b64`` string + (JSON cannot hold raw bytes, so the body is base64-encoded and decoded verbatim + on read — a byte-exact round-trip). +* ``created_at`` (inherited, timezone-aware) is persisted from the record's own + ``StoredResponse.created_at`` rather than the DB clock, so a round-tripped entry + equals the one the middleware stored. +* ``expires_at`` (timezone-aware) is the wall-clock TTL horizon (``created_at`` / + reserve-time ``+ ttl``); a read past it is treated as absent, and the + reserve path overwrites it — the durable analogue of the in-memory store's + monotonic-clock eviction. Indexed so an expiry sweep touches only stale rows. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import DateTime, Index, String +from sqlalchemy.orm import Mapped, mapped_column + +from forge_db.base import ForgeModel, json_type + + +class IdempotencyKey(ForgeModel): + """One cached HTTP response keyed by a tenant-scoped idempotency token. + + Not workspace-scoped: the tenant is already baked into ``key`` (the middleware + hashes the presented credential into it), and an anonymous request scopes by + client IP — there is no ``workspace`` FK to hang it on, so this uses the plain + :class:`ForgeModel` (surrogate UUID PK + timestamps) like + :class:`~forge_db.models.sso.SamlReplay`. + """ + + __tablename__ = "idempotency_key" + __table_args__ = ( + # Expiry sweep / read-time-expiry queries touch only rows near their TTL. + Index("ix_idempotency_key_expires_at", "expires_at"), + ) + + #: The middleware's fully-qualified, tenant-scoped cache key. UNIQUE (not PK) + #: per the house UUID-PK invariant; it is the ``ON CONFLICT`` target. + key: Mapped[str] = mapped_column(String(512), nullable=False, unique=True) + #: JSONB image of the ``StoredResponse`` (``request_hash`` / ``status_code`` / + #: ``content_type`` / base64 ``body_b64``); the body is opaque bytes. + response: Mapped[dict[str, Any]] = mapped_column(json_type(), nullable=False) + #: Wall-clock TTL horizon; a read past it is absent, the reserve path overwrites. + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover - trivial + return ( + f"IdempotencyKey(id={self.id!r}, key={self.key!r}, " + f"expires_at={self.expires_at!r})" + ) + + +__all__ = ["IdempotencyKey"] diff --git a/packages/db/migrations/versions/0028_idempotency_store.py b/packages/db/migrations/versions/0028_idempotency_store.py new file mode 100644 index 00000000..120d983c --- /dev/null +++ b/packages/db/migrations/versions/0028_idempotency_store.py @@ -0,0 +1,71 @@ +"""idempotency store: HTTP idempotency-key response-cache ``idempotency_key`` table. + +Backs the *db* variant of the API's HTTP idempotency store +(:class:`forge_api.middleware.idempotency_db.DbIdempotencyStore` — the storage +boundary behind ``IdempotencyMiddleware``) with real Postgres persistence. +Creates one new, self-contained table: + +* ``idempotency_key`` — one row per cached response: the tenant-scoped ``key`` + (UNIQUE — the reserve/get-or-set ``ON CONFLICT`` target), the JSONB + ``response`` image of the ``StoredResponse`` (``request_hash`` / ``status_code`` + / ``content_type`` / base64 ``body_b64``), the domain record's own tz-aware + ``created_at``, and the tz-aware ``expires_at`` TTL horizon (indexed for expiry + sweeps). + +Distinct from the per-domain ``automation_execution.idempotency_key`` / +``deployment.idempotency_key`` columns (service-layer command dedup): this is the +transport-level HTTP response cache the middleware owns, so this revision only +*adds* a table and touches nothing existing. + +Foundation note (mirrors 0024/0025/0027): ``forge_db``'s metadata is the source +of truth, so a fresh chain already provisions this table from the model. To stay +idiomatic *and* own an explicit, reversible step this migration is idempotent: +``upgrade`` creates only what is missing, ``downgrade`` drops only what this +revision introduced. Applies cleanly on SQLite (unit path) and pgvector Postgres. + +Revision ID: 0028_idempotency_store +Revises: 0027_secret_vault_store +Create Date: 2026-07-05 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) +from forge_db.base import Base + +# revision identifiers, used by Alembic. +revision: str = "0028_idempotency_store" +down_revision: str | None = "0027_secret_vault_store" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +# Tables owned by this revision (downgrade drops them in reverse). +_TABLES: tuple[str, ...] = ("idempotency_key",) + + +def _owned_tables() -> list[sa.Table]: + by_name = {t.name: t for t in Base.metadata.sorted_tables} + return [by_name[name] for name in _TABLES if name in by_name] + + +def _existing_tables() -> set[str]: + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def upgrade() -> None: + existing = _existing_tables() + to_create = [t for t in _owned_tables() if t.name not in existing] + if to_create: + Base.metadata.create_all(bind=op.get_bind(), tables=to_create) + + +def downgrade() -> None: + existing = _existing_tables() + to_drop = [t for t in reversed(_owned_tables()) if t.name in existing] + if to_drop: + Base.metadata.drop_all(bind=op.get_bind(), tables=to_drop) diff --git a/packages/db/tests/test_migration.py b/packages/db/tests/test_migration.py index 7fa5734a..26be98ab 100644 --- a/packages/db/tests/test_migration.py +++ b/packages/db/tests/test_migration.py @@ -1033,6 +1033,42 @@ def test_approval_repository_columns_migration_up_down(alembic_config: Config) - engine.dispose() +# HARD-11-persist HTTP idempotency-store table, owned by 0028_idempotency_store. +IDEMPOTENCY_TABLES = {"idempotency_key"} + + +def test_idempotency_store_migration_up_down(alembic_config: Config) -> None: + """0028 owns the ``idempotency_key`` response-cache table (with its ``key`` + UNIQUE index + ``expires_at`` index) and drops exactly it on downgrade, leaving + the prior chain intact. + + (forge_db's baseline is metadata-driven, so a fresh chain provisions the table + at 0001; like 0025/0027, the 0028 step is idempotent about that and owns a + clean, reversible down.)""" + url = alembic_config.get_main_option("sqlalchemy.url") + assert url is not None + engine = create_engine(url) + try: + command.upgrade(alembic_config, "head") + inspector = inspect(engine) + tables = set(inspector.get_table_names()) + assert tables >= IDEMPOTENCY_TABLES, "missing idempotency_key table" + indexes = {i["name"] for i in inspector.get_indexes("idempotency_key")} + assert "ix_idempotency_key_expires_at" in indexes + uniques = {uc["name"] for uc in inspector.get_unique_constraints("idempotency_key")} + assert "uq_idempotency_key_key" in uniques + + # Downgrade one step: the idempotency table is gone, the chain intact. + command.downgrade(alembic_config, "0027_secret_vault_store") + after_down = set(inspect(engine).get_table_names()) + assert not (IDEMPOTENCY_TABLES & after_down), "downgrade left idempotency_key" + assert after_down >= EXPECTED_TABLES + + command.downgrade(alembic_config, "base") + finally: + engine.dispose() + + # --------------------------------------------------------------------------- # # HARD-11 — live-Postgres migration round-trip, per-revision walk, and # # data-preservation. These run only against a real pgvector Postgres (the # diff --git a/packages/db/tests/test_models.py b/packages/db/tests/test_models.py index 4e808e7f..b1e22964 100644 --- a/packages/db/tests/test_models.py +++ b/packages/db/tests/test_models.py @@ -144,6 +144,10 @@ # store (the storage boundary behind ``SecretVault``). Workspace-scoped; # holds only ciphertext, never plaintext. "Secret", + # idempotency-store persistence: durable backing for the HTTP idempotency-key + # response cache (the store behind ``IdempotencyMiddleware``). Not tenant- + # scoped — the tenant is baked into the ``key`` (anonymous scopes by client IP). + "IdempotencyKey", ] # Tables that are NOT the tenant root and therefore must carry a workspace FK. @@ -167,6 +171,10 @@ # ``workspace_ref`` tag, and the cursor row no workspace column at all. "observability_audit_entry", "observability_audit_chain_head", + # The HTTP idempotency-key response cache is keyed by a tenant-scoped string + # (``forge:idem::
``, anonymous falls back to client IP), so the + # tenant lives in the key itself — there is no ``workspace`` FK to hang it on. + "idempotency_key", } From a0ca9c1412709a6825f525d34c2da3d1111587e7 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 18:17:05 +0200 Subject: [PATCH 11/12] docs(db): PERSISTENCE_PROGRESS ledger for 10 Postgres-backed stores Records the in-memory -> Postgres persistence pass: per-target refuted/repaired/decision table, target->repo->migration->test map, the five chained Alembic migrations (0024-0028), the eleven FORGE_*_BACKEND env flags (all default memory), and green-gate proof (ruff clean; 3688 passed / 53 skipped on pgvector :5433). Co-Authored-By: Claude Opus 4.8 --- docs/PERSISTENCE_PROGRESS.md | 123 +++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 docs/PERSISTENCE_PROGRESS.md diff --git a/docs/PERSISTENCE_PROGRESS.md b/docs/PERSISTENCE_PROGRESS.md new file mode 100644 index 00000000..94aad340 --- /dev/null +++ b/docs/PERSISTENCE_PROGRESS.md @@ -0,0 +1,123 @@ +# Persistence Progress — In-Memory Stores → Real Postgres + +Goal: give in-memory service stores **real Postgres persistence without changing +behaviour**. For each target a Postgres-backed repository was implemented that +satisfies the **same frozen Protocol** the InMemory store already implements. The +InMemory store **stays** and remains the **default** so every existing unit test +runs untouched (hermetic, no Postgres). The backend is chosen at the composition +root by an env flag, `FORGE__BACKEND=db|memory`, **default `memory`**. + +Method per target (TDD, PARK-DON'T-FAKE): stand up the DB repo against the +canonical `forge_db` ORM (extend, never fork), add a chained Alembic migration +where a new column/table was needed, wire the seam behind the env flag, and add a +DB-backed integration test that exercises the Protocol against real Postgres +(round-trip, filtering, ordering, constraints). Nothing was weakened or skipped +to go green. + +## Results + +| id | refuted | repaired | decision | +|----|---------|----------|----------| +| board-service | 0 | no | committed | +| audit-store | 0 | no | committed | +| approval-repository | 0 | no | committed | +| policy-override-grant-store | 0 | no | committed | +| api-key-backend | 0 | no | committed | +| policy-audit-sink | 0 | no | committed | +| spec-projection-repository | 0 | no | committed | +| pm-link-repository | 0 | no | committed | +| secret-vault-store | 0 | no | committed | +| idempotency-store | 0 | no | committed | + +`refuted` = integration assertions that failed and forced a code change during +TDD; `repaired` = whether the InMemory store itself had to be touched (it never +did — the Protocol was the only shared surface). All ten are committed as +individual `feat(db/): Postgres persistence` commits. + +## Target → repo → migration → test map + +| target | Protocol (frozen) | Postgres repo | Alembic migration | table(s) | env flag | DB integration test | +|--------|-------------------|---------------|-------------------|----------|----------|---------------------| +| board-service | `forge_contracts.BoardService` | `packages/board-core/forge_board/sql_service.py` (`SqlAlchemyBoardService`) | `0024_board_persistence` (add columns) | `task`, `epic`, `sprint`, `spec`, `task_dependency` (existing F-series tables extended) | `FORGE_BOARD_BACKEND` | `packages/board-core/tests/test_sql_board_service.py` | +| audit-store | `forge_contracts.AuditSink` | `apps/api/forge_api/observability/audit_db.py` | `0025_observability_audit_store` (new) | `observability_audit_entry`, `observability_audit_chain_head` | `FORGE_AUDIT_BACKEND` (MCP bridge: `FORGE_MCP_AUDIT_BACKEND`) | `apps/api/tests/test_audit_store_db.py` | +| approval-repository | `ApprovalRepository` (F36) | `apps/api/forge_api/services/approval_repository_db.py` | `0026_approval_repository_columns` (add columns) | `approval_request` (existing, extended) | `FORGE_APPROVAL_BACKEND` | `apps/api/tests/test_approval_repository_db.py` | +| policy-override-grant-store | `GrantStore` (F36 J5) | `apps/api/forge_api/services/policy_override_grant_store_db.py` | none — reuses existing table | `policy_override_grant` (existing) | `FORGE_OVERRIDE_GRANT_BACKEND` | `apps/api/tests/test_policy_override_grant_store_db.py` | +| api-key-backend | API-key backend Protocol | `apps/api/forge_api/auth/apikeys_db.py` | none — reuses existing table | `platform_api_key` (existing) | `FORGE_APIKEY_BACKEND` | `apps/api/tests/test_apikeys_db.py` | +| policy-audit-sink | Policy audit sink Protocol (F29) | `apps/api/forge_api/services/policy_audit_sink_db.py` | none — reuses existing table | `policy_rule_evaluation` (existing) | `FORGE_POLICY_AUDIT_BACKEND` | `apps/api/tests/test_policy_audit_sink_db.py` | +| spec-projection-repository | Projection repository Protocol (F23) | `apps/api/forge_api/services/projection_repository_db.py` | none — reuses existing tables | `traceability_spec_rollup`, `traceability_criterion_link` (existing) | `FORGE_PROJECTION_BACKEND` | `apps/api/tests/test_projection_repository_db.py` | +| pm-link-repository | Link repository Protocol (F18) | `apps/api/forge_api/services/pm_link_repository_db.py` | none — reuses existing table | `pm_task_link` (existing) | `FORGE_PM_LINK_BACKEND` | `apps/api/tests/test_pm_link_repository_db.py` | +| secret-vault-store | `forge_contracts.Vault` / secret store | `apps/api/forge_api/auth/vault_db.py` | `0027_secret_vault_store` (new) | `secret` | `FORGE_SECRET_BACKEND` | `apps/api/tests/test_secret_vault_store_db.py` | +| idempotency-store | Idempotency store Protocol | `apps/api/forge_api/middleware/idempotency_db.py` | `0028_idempotency_store` (new) | `idempotency_key` | `FORGE_IDEMPOTENCY_BACKEND` | `apps/api/tests/test_idempotency_store_db.py` | + +## Which services are now Postgres-backed vs still memory-only + +All ten targets are now **Postgres-capable**: each has a real Postgres-backed +repository selectable at the composition root. They remain **memory-default** — +`db` is opt-in per area — so unit tests stay hermetic. Concretely: + +- **Postgres-backed available (opt-in via env flag), memory default:** board + service, observability audit store, approval repository, policy-override grant + store, platform API-key backend, policy-audit sink, spec/traceability + projection repository, PM-sync link repository, encrypted secret-vault store, + HTTP idempotency store. +- **Still memory-only (out of scope for this pass):** every other in-memory seam + not listed above keeps its InMemory implementation with no DB alternative yet + (e.g. worker task-dedup runs behind its own separate `FORGE_TASK_DEDUP_BACKEND` + flag and was not part of this batch). + +The InMemory implementation of each target is retained and is the default, so no +existing test needed modification. + +## Alembic migrations added + +Chained sequentially onto the prior head `0023_envelope_key_version`; each has +both `upgrade()` and `downgrade()` and applies cleanly on the `:5433` pgvector +test DB. Five of the ten targets required schema; the other five persist against +tables already created by earlier F-series migrations. + +| revision | down_revision | change | +|----------|---------------|--------| +| `0024_board_persistence` | `0023_envelope_key_version` | adds board columns to `task`/`epic`/`sprint`/`spec`/`task_dependency` | +| `0025_observability_audit_store` | `0024_board_persistence` | new `observability_audit_entry` + `observability_audit_chain_head` (hash-chained audit) | +| `0026_approval_repository_columns` | `0025_observability_audit_store` | adds columns to `approval_request` | +| `0027_secret_vault_store` | `0026_approval_repository_columns` | new `secret` table (envelope-encrypted vault) | +| `0028_idempotency_store` | `0027_secret_vault_store` | new `idempotency_key` table | + +Current Alembic head: **`0028_idempotency_store`**. + +## Env flags added (composition root) + +All read in `apps/api/forge_api/settings.py` (each defaults to `memory`): + +| flag | selects | +|------|---------| +| `FORGE_BOARD_BACKEND` | board service (`memory` → `InMemoryBoardService`, `db` → `SqlAlchemyBoardService`) | +| `FORGE_AUDIT_BACKEND` | observability audit store | +| `FORGE_MCP_AUDIT_BACKEND` | MCP audit bridge (forwards to the audit store) | +| `FORGE_APPROVAL_BACKEND` | approval repository | +| `FORGE_OVERRIDE_GRANT_BACKEND` | policy-override grant store | +| `FORGE_APIKEY_BACKEND` | platform API-key backend | +| `FORGE_POLICY_AUDIT_BACKEND` | policy-audit sink | +| `FORGE_PROJECTION_BACKEND` | spec/traceability projection repository | +| `FORGE_PM_LINK_BACKEND` | PM-sync link repository | +| `FORGE_SECRET_BACKEND` | encrypted secret-vault store | +| `FORGE_IDEMPOTENCY_BACKEND` | HTTP idempotency store | + +## Parked items + +- None functional. The only prior parked note — that the whole-repo + `uv run pytest -q` had not been confirmed green because the earlier report was + forced while pytest was still buffering — is **closed**: the full suite was + re-run under `FORGE_TEST_DATABASE_URL` (pgvector `:5433`) and is green (see + Green Gate below). + +## Green gate + +- `uv run ruff check .` → **All checks passed!** +- `FORGE_TEST_DATABASE_URL=postgresql+psycopg://forge:forge@localhost:5433/forge uv run pytest -q` + → **3688 passed, 53 skipped, 23 warnings in 667.85s** (exit 0). + +The 53 skips are pre-existing live-lane tests that skip cleanly when their +external creds/binaries are absent (live MCP transport, live GitHub webhook +secret, live model provider, `promtool`/`amtool` on PATH) — none were introduced +or weakened by this work. From 6f4df98dbab61b09deb2be285bb04a4fc83c94f3 Mon Sep 17 00:00:00 2001 From: Forge Swarm Date: Sun, 5 Jul 2026 18:21:01 +0200 Subject: [PATCH 12/12] docs: use service-hive/forge org URLs --- README.md | 4 ++-- docs/FORGE_SPEC.md | 4 ++-- docs/implementation-slices/v1/F13-local-quickstart.md | 2 +- docs/implementation-slices/v1/F14-docker-compose-selfhost.md | 2 +- docs/runbooks/live-github.md | 2 +- docs/self-hosting/quickstart.md | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6baaf6c6..33c65c6b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ > a sandboxed agent runtime, hybrid knowledge retrieval, and a native project > board, all on one Postgres-backed platform you run yourself. - + [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](./LICENSE) [![CI](https://img.shields.io/badge/CI-see%20Actions-lightgrey.svg)](../../actions) [![Status: pre-1.0](https://img.shields.io/badge/status-pre--1.0%20(active)-orange.svg)](#status) @@ -65,7 +65,7 @@ for the honest per-feature ledger. Requires Docker Engine 24+ and the Docker Compose v2 plugin, plus `make`. ```bash -git clone https://github.com//forge.git +git clone https://github.com/service-hive/forge.git cd forge cp .env.example .env # then set SECRET_KEY, POSTGRES_PASSWORD, DOMAIN, ... make dev # build + start the full stack, migrate, seed, wait healthy diff --git a/docs/FORGE_SPEC.md b/docs/FORGE_SPEC.md index 08587c1f..0b3720d8 100644 --- a/docs/FORGE_SPEC.md +++ b/docs/FORGE_SPEC.md @@ -820,7 +820,7 @@ Forge is self-hosting-first. Full feature parity for self-hosted deployments. ### Local Quickstart ```bash -git clone https://github.com/forge-platform/forge +git clone https://github.com/service-hive/forge cd forge cp .env.example .env make setup # installs deps, runs migrations, seeds demo workspace @@ -836,7 +836,7 @@ Target: Ubuntu 24.04 LTS — minimum 4 vCPU / 8 GB RAM / 50 GB SSD curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER && newgrp docker -git clone https://github.com/forge-platform/forge +git clone https://github.com/service-hive/forge cd forge cp .env.production.example .env.production # Fill in: SECRET_KEY, DB_PASSWORD, GITHUB_APP_*, MODEL_PROVIDER_KEY, DOMAIN diff --git a/docs/implementation-slices/v1/F13-local-quickstart.md b/docs/implementation-slices/v1/F13-local-quickstart.md index c6c56e39..43b8e0d1 100644 --- a/docs/implementation-slices/v1/F13-local-quickstart.md +++ b/docs/implementation-slices/v1/F13-local-quickstart.md @@ -23,7 +23,7 @@ This slice is **mechanism + content-orchestration + docs**, not new product surf The canonical journey (mirrors the spec's "Local Quickstart" block verbatim): ```bash -git clone https://github.com/forge-platform/forge +git clone https://github.com/service-hive/forge cd forge make setup # preflight, install deps, bring up infra, migrate, seed demo workspace make dev # start all services diff --git a/docs/implementation-slices/v1/F14-docker-compose-selfhost.md b/docs/implementation-slices/v1/F14-docker-compose-selfhost.md index 13b47d99..3a0ab8cc 100644 --- a/docs/implementation-slices/v1/F14-docker-compose-selfhost.md +++ b/docs/implementation-slices/v1/F14-docker-compose-selfhost.md @@ -33,7 +33,7 @@ The "user" here is a **self-hosting operator** (admin running Forge on their own curl -fsSL https://get.docker.com | sh sudo usermod -aG docker $USER && newgrp docker -git clone https://github.com/forge-platform/forge && cd forge +git clone https://github.com/service-hive/forge && cd forge sudo cp deploy/docker/daemon.json /etc/docker/daemon.json && sudo systemctl restart docker # log caps cp deploy/.env.production.example .env.production $EDITOR .env.production # fill SECRET_KEY, AUTH_SECRET, API_KEY_PEPPER, FORGE_VAULT_KEYS, INTERNAL_SERVICE_TOKEN, diff --git a/docs/runbooks/live-github.md b/docs/runbooks/live-github.md index e12e30aa..8493aa73 100644 --- a/docs/runbooks/live-github.md +++ b/docs/runbooks/live-github.md @@ -32,7 +32,7 @@ It closes the GitHub half of release blocker #1 (BETA gate **G-GH**). 3. Set a **Webhook secret** (a long random string). Record it. 4. **Generate a private key** → downloads a `.pem`. This file is the only long-lived secret; treat it like a password. -5. **Install** the App on a *disposable* test repo (e.g. `your-org/forge-ci-sandbox`) +5. **Install** the App on a *disposable* test repo (e.g. `service-hive/forge-ci-sandbox`) and note the **installation id** (the numeric id in the install URL, or via `GET /app/installations` with an App JWT). diff --git a/docs/self-hosting/quickstart.md b/docs/self-hosting/quickstart.md index 973bc48c..b73058b5 100644 --- a/docs/self-hosting/quickstart.md +++ b/docs/self-hosting/quickstart.md @@ -18,7 +18,7 @@ API. For the production hardening details see ## 1. Clone and configure ```bash -git clone https://github.com/your-org/forge.git +git clone https://github.com/service-hive/forge.git cd forge cp .env.example .env ```