diff --git a/.env.example b/.env.example index 06661bb6..31139896 100644 --- a/.env.example +++ b/.env.example @@ -65,11 +65,12 @@ DATABASE_URL=postgresql+psycopg://forge:change-me@localhost:5432/forge # Redis (queue, cache, sessions) REDIS_URL=redis://localhost:6379/0 -# Object storage (MinIO / S3-compatible) +# Object storage (MinIO / S3-compatible). Only the root credentials are consumed +# today — by the compose `minio` service (`${MINIO_ROOT_USER}`/`${MINIO_ROOT_PASSWORD}`). +# Endpoint/bucket wiring is PARKED with the object-storage scope: no code reads +# MINIO_ENDPOINT / MINIO_BUCKET yet, so they are omitted until they are wired. MINIO_ROOT_USER=forge MINIO_ROOT_PASSWORD=change-me -MINIO_ENDPOINT=http://localhost:9000 -MINIO_BUCKET=forge-artifacts # Auth (Better Auth / Auth.js) AUTH_SECRET=change-me-generate-a-long-random-string @@ -86,13 +87,30 @@ API_KEY_PEPPER= # Beat cadence for the expired platform-key purge (seconds; default 15m). FORGE_AUTH_PURGE_KEYS_INTERVAL_SECONDS=900 -# BYOK model provider (provider-agnostic; Anthropic reference impl) -MODEL_PROVIDER=anthropic -MODEL_PROVIDER_KEY= -EMBEDDING_PROVIDER=anthropic -EMBEDDING_MODEL= -RERANKER_URL=http://localhost:8080 -RERANKER_MODEL=jina-reranker-v2-base-multilingual +# --------------------------------------------------------------------------- +# BYOK model provider (provider-agnostic; Anthropic reference impl). UNSET by +# default: with no FORGE_MODEL_PROVIDER the worker runs the offline, deterministic +# ScriptedModelClient (canned output, no network) and logs a WARNING each run. +# Uncomment + fill these in to drive a real provider. +# --------------------------------------------------------------------------- +# Master switch: anthropic | openai. Absent -> offline scripted fallback. +# FORGE_MODEL_PROVIDER=anthropic +# Model name. Optional for anthropic (a reference default is applied); REQUIRED +# for openai. +# FORGE_MODEL_NAME= +# BYOK key. The provider-native ANTHROPIC_API_KEY / OPENAI_API_KEY takes +# precedence; FORGE_MODEL_API_KEY is the provider-agnostic fallback. +# FORGE_MODEL_API_KEY= + +# Multi-agent coordinator (supervised runs). Required `true` to enable; when false, +# supervised runs escalate to a human with reason `multi_agent_disabled`. +# MULTI_AGENT_ENABLED=false + +# Embeddings for hybrid-retrieval evaluation (OpenAI-compatible embedder), shown +# with the built-in defaults; override to point at another endpoint. +# EMBEDDING_MODEL=text-embedding-3-small +# EMBEDDING_BASE_URL=https://api.openai.com/v1 +# EMBEDDING_DIM=1536 # HARD-03 live cross-encoder reranker (BYOK). Provider `fixture` (default) keeps # the offline deterministic reranker; `jina`/`cohere`/`selfhosted` build a @@ -131,8 +149,7 @@ FORGE_GRAFANA_WEBHOOK_SECRET= # in the bundled incident.yaml, not an env var). FORGE_INCIDENT_RECOVERY_WINDOW_SECONDS=300 FORGE_INCIDENT_RECOVERY_MAX_WINDOWS=6 -# MinIO bucket for versioned postmortem snapshots. -FORGE_POSTMORTEMS_BUCKET=forge-postmortems +# Postmortem-snapshot bucket: PARKED with the object-storage scope (no reader yet). # External PM adapters (F18 — Jira, Linear). OAuth client creds are optional # (api_token auth works without them). Webhook bodies for these routes MUST reach @@ -177,7 +194,24 @@ FORGE_SANDBOX_MAX_TTL_SECONDS=21600 # Service URLs (used by web / inter-service calls) API_URL=http://localhost:8000 MCP_GATEWAY_URL=http://localhost:8001 -NEXT_PUBLIC_API_URL=http://localhost:8000 +# Browser REST API base. Inlined into the web bundle at BUILD time (NEXT_PUBLIC_*), +# so a runtime env var is too late for the client. Leave UNSET for the common +# cases: the client auto-derives a same-origin `https:///api` in the +# browser (the edge strips `/api` and routes it to the API — see +# docs/self-hosting/reverse-proxy.md), and keeps `http://localhost:8000` on the +# local `next dev` server (:3000). Set it ONLY when the API is NOT same-origin — +# an absolute value (e.g. `https://api.example.com`) is used verbatim, a relative +# value (e.g. `/api`) resolves against the page origin — and set it as a BUILD +# arg; a runtime env var alone is too late. +# NEXT_PUBLIC_API_URL=http://localhost:8000 +# Realtime (board push + spec co-editing) WebSocket endpoint. Inlined into the +# web bundle at BUILD time (NEXT_PUBLIC_*). Leave UNSET for the common cases: +# the client auto-derives a same-origin `wss:///ws` in the browser +# (which the edge routes to the API — see docs/self-hosting/reverse-proxy.md), +# and keeps `ws://localhost:8000/ws` on the local `next dev` server (:3000). +# Set it ONLY when the WS endpoint is NOT same-origin (e.g. a dedicated realtime +# host), and set it as a BUILD arg — a runtime env var alone is too late. +# NEXT_PUBLIC_WS_URL=wss://forge.example.com/ws # MCP sync-and-index (F20) — periodic ingestion of MCP resources into the index. MCP_INDEX_POLL_SECONDS=300 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66d2a1e5..bdf15c89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,7 +241,7 @@ jobs: run: uv run pytest -m live_slack -q web: - name: web (lint + build) + name: web (lint + types + tests + build) runs-on: ubuntu-latest steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -263,6 +263,16 @@ jobs: - name: Lint run: pnpm -r lint + # The web package ships 89 vitest files and a `tsc --noEmit` typecheck that + # previously only ran in the (unreachable) release workflow. Gate both on + # every PR/push, after lint and before build, so a type error or a broken + # component test blocks the merge instead of shipping. + - name: Type-check (tsc) + run: pnpm --filter @forge/web typecheck + + - name: Test (vitest) + run: pnpm --filter @forge/web test + - name: Build run: pnpm -r build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08a0a42e..f301d7c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,15 +189,24 @@ jobs: cp CHANGELOG.md RELEASE_NOTES.md # --- Publish the GitHub Release (gh CLI, GITHUB_TOKEN) --- # + # Draft by default: a tag push always creates a draft (the `inputs` + # context is empty on a non-workflow_dispatch trigger, so the first + # disjunct short-circuits `true`); only an explicit `workflow_dispatch` + # run with draft=false produces a published (non-draft) release. - name: Create GitHub Release if: startsWith(github.ref, 'refs/tags/v') env: GH_TOKEN: ${{ github.token }} + RELEASE_IS_DRAFT: ${{ (github.event_name != 'workflow_dispatch') || inputs.draft }} run: | + DRAFT_FLAG="" + if [ "$RELEASE_IS_DRAFT" = "true" ]; then + DRAFT_FLAG="--draft" + fi gh release create "${GITHUB_REF_NAME}" \ --title "${GITHUB_REF_NAME}" \ --notes-file RELEASE_NOTES.md \ - --draft \ + $DRAFT_FLAG \ release/sbom/forge-source.cdx.json \ deploy/sbom/*.cdx.json \ deploy/build-manifest.json \ diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 7f58c157..43829b81 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -5,6 +5,15 @@ # # Third-party (and official actions/*) actions are pinned to a full commit SHA # with a human-readable version comment, so a moved tag cannot alter what runs. +# +# NOTE — intentional overlap with ci.yml's `security` job: that job runs the +# same scanners (bandit, pip-audit, gitleaks) plus semgrep + SBOM + the +# enforcement-matrix suite as part of the full green gate. This workflow is +# kept as a separate, minimal-permission (contents: read only), fast-to-review +# check so the core secret/SAST/dependency gate can be reasoned about (and +# required in branch protection) independently of the rest of CI. Do not +# de-duplicate one into the other without checking branch-protection required +# status checks first. name: security on: diff --git a/CHANGELOG.md b/CHANGELOG.md index 551ddf0a..c070f840 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,42 @@ sections by hand; write a well-formed commit instead. ### Added +- **self-eval**: enforce the gate on config changes + worker-driven run (Phase A wiring) (#66) +- **self-eval**: baseline persistence + live agent-backed eval runner (Phase A substrate) (#65) +- Self-Eval Gate — block config changes that regress a workspace's private per-repo suite (trust-layer 4/4) (#64) +- Red-Team Gate (trust layer, phase 3) — adversary must break the change in-sandbox before the human gate (#63) +- Time-Travel Runs (trust layer, phase 2) — deterministic record-replay of agent runs (#62) +- Attested Changesets (trust layer, phase 1) — signed provenance chained into the tamper-evident audit log (#61) +- OIDC SSO + marketplace publish + benchmark leaderboard + under-dev banner (#57) +- frontend UX pass — clearer IA/nav, one primary action, progressive disclosure, empty/loading/error states, a11y (#39) +- IaC — OpenTofu infra/ (Hetzner control-plane + Cloudflare + Fly agents), dev/staging/prod, remote state, runbook (#38) +- F40 deferred-scope deltas — PM adapters (BYO board), MCP, policy, automations, sprint depth, observability (#37) +- realtime co-editing (WS server + CRDT spec co-editing + live push) (#36) +- Spec Studio — dual-format spec authoring (Guided/Markdown/YAML/Read), BYOK AI draft, lifecycle, versioning (#33) +- adaptive orchestration (auto model routing + per-role effort + settings + cost-by-tier) (#32) +- public-readiness — under-dev banner, honest status, live spec dashboard (#30) +- **web/walkthrough**: In-app guided walkthrough +- **web/workflow-editor**: Workflow visual editor +- **web/pm-integrations**: PM integrations +- **web/rbac-admin**: Multi-team & RBAC admin +- **web/sso-settings**: SSO / SCIM settings +- **web/deployment-gates**: Deployment gates +- **web/audit-log**: Audit viewer +- **web/sprints**: Sprints & velocity +- **web/observability**: Observability & cost +- **web/incidents**: Incidents +- **web/marketplace**: Marketplace +- **web/spec-dashboard**: Spec-validation dashboard +- **web/run-trace-viewer**: Run-trace viewer +- **web/approval-inbox**: Approval inbox +- **web/board-depth**: Board depth +- **HARD-06**: live-slack +- **HARD-05**: live-mcp-server +- **HARD-03**: live-reranker +- **HARD-02**: live-model-byok +- **HARD-01**: live-github-app +- **HARD-08**: kubernetes-helm-deploy +- **HARD-12**: release-engineering - **HARD-10**: observability-cost-prod - **HARD-04**: real-eval-corpus - **HARD-11**: reliability-maturity @@ -62,3 +98,7 @@ sections by hand; write a well-formed commit instead. - **phase0**: 0.3 contracts (packages/contracts) - **phase0**: 0.2 data-model (packages/db) - **phase0**: 0.1 workspace+tooling + +### Fixed + +- **config**: document real FORGE_MODEL_* env vars and warn on scripted-client fallback diff --git a/Makefile b/Makefile index 75a59b29..e595ad4b 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ MYPY_PACKAGES := \ forge_contracts forge_db forge_workflow forge_agent forge_coordinator \ forge_spec forge_board forge_knowledge forge_integrations forge_mcp \ forge_policy forge_authz forge_skill forge_eval forge_approval forge_api \ - forge_worker forge_mcp_gateway forge_orchestration_policy + forge_worker forge_mcp_gateway forge_orchestration_policy \ + forge_deploy forge_auth forge_marketplace forge_obs help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ @@ -125,8 +126,10 @@ hooks: ## Install the commit-msg hook that enforces Conventional Commits (cz che @chmod +x .git/hooks/commit-msg @echo "Installed .git/hooks/commit-msg (uv run cz check)." -release-readiness: ## Run the automated RELEASE_READINESS gate at the PRODUCTION bar - uv run forge-release-readiness --bar production +BAR ?= beta + +release-readiness: ## Run the automated RELEASE_READINESS gate at BAR (default: beta; override with BAR=production) + uv run forge-release-readiness --bar $(BAR) source-sbom: ## Generate the source-tree CycloneDX SBOM (release/sbom/forge-source.cdx.json) release/scripts/source-sbom.sh diff --git a/README.md b/README.md index b2afc77a..cda58790 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ > ⚠️ **Under active development — pre-1.0, not production-ready.** Forge is shared > openly for **evaluation and testing**, not for production use yet. Expect rough > edges, changing APIs, and features that are API/CLI-first with their UI or live -> integrations still landing. Read **[Status](#status)** for the honest per-area +> integrations pending. Read **[Status](#status)** for the honest per-area > state before you rely on it, and please contribute via pull request. [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](./LICENSE) @@ -36,12 +36,13 @@ Forge is **pre-1.0 and under active development** — usable for evaluation and self-host testing, **not yet for production**. The backend platform, HTTP API, CLI, workflow/agent runtime, and self-hosting substrate are the mature surface, exercised by a large test suite (~3,700 tests on real pgvector Postgres, green -in CI). The **web UI ships 15 feature screens** (board, approvals, run-trace -viewer, spec dashboard, marketplace, incidents, observability, sprints, audit, -deployment gates, SSO/SCIM, RBAC admin, PM integrations, workflow editor, and a -guided walkthrough) on the Forge design system. Some screens carry **honestly -marked gaps** where a backend projection or live credential is still landing -(e.g. a couple of dashboard projections, OIDC), and the +in CI). The **web UI ships 15 feature screens** — `approvals`, `audit`, +`board`, `deployments`, `depth`, `incidents`, `leaderboard`, `marketplace`, +`observability`, `runs`, `settings` (SSO/OIDC/SCIM, RBAC, PM integrations, +model BYOK), `specs`, `sprints`, `walkthrough`, and `workflow` — on the Forge +design system. Some screens carry **honestly marked gaps** where a backend +projection or live credential is pending (e.g. a couple of dashboard +projections), and the third-party integrations (GitHub App, model BYOK, reranker, MCP, Slack) are code-complete with tests + runbooks but need **your keys** to verify live. We try hard not to advertise anything that is only parked — the pre-1.0 notice at @@ -54,9 +55,10 @@ individual screens mark in-progress areas inline. - **Spec-driven development** — author a `manifest.yaml` spec; the spec engine validates it and drives the work. Includes a spec-validation dashboard. - **Agent runtime** — a LangGraph agent loop that runs work inside sandboxed - execution (Docker today; gVisor / Firecracker isolation classes are modelled - and mapped, with the real-runtime tiers gated behind a virtualization-enabled - CI job). + execution (`worktree` git-worktree isolation by default, a per-task Docker + `container` sandbox also available; gVisor / Firecracker isolation classes + are modelled and mapped, with the real-runtime tiers gated behind a + virtualization-enabled CI job). - **Multi-agent coordination** — a coordinator for fanning work across agents. - **Workflow engine** — a Postgres finite-state-machine workflow layer, with Temporal available in the production stack for durable orchestration. @@ -65,13 +67,14 @@ individual screens mark in-progress areas inline. - **Native project board** — a board core for tracking runs and work items. - **Policy, skill, integration & MCP SDKs** — declarative `.forge/policy.yaml`, skill profiles, integration definitions, and an MCP gateway for tool sources. -- **Integration marketplace** — browse and install integrations (UI shipped; - publishing still via the offline author CLI). -- **Enterprise SSO / SCIM** — SAML SSO and SCIM provisioning with an admin UI - (OIDC and live IdP verification still landing). +- **Integration marketplace** — browse, install, and publish integrations + from the in-app UI (the offline `forge marketplace package` CLI works too). +- **Enterprise SSO / SCIM** — SAML SSO and OIDC, plus SCIM provisioning, with + an admin UI (live IdP verification needs your own IdP, like the third-party + integrations above). - **Human approval system** — gated approvals for sensitive agent actions. -- **Benchmark leaderboard** — submit, verify, and rank agent benchmark runs - (backend; **UI in progress**). +- **Benchmark leaderboard** — submit, verify, and rank agent benchmark runs, + with a public leaderboard UI. - **Auth, secrets & BYOK** — envelope-encrypted secrets, a key vault, and bring-your-own-key model-provider credentials. - **Observability & cost metrics + audit log** — structured, redaction-aware @@ -133,6 +136,7 @@ forge/ │ ├── workflow-engine/ # forge_workflow │ ├── agent-runtime/ # forge_agent │ ├── multi-agent-coordinator/ # forge_coordinator +│ ├── orchestration-policy/ # forge_orchestration_policy │ ├── spec-engine/ # forge_spec │ ├── board-core/ # forge_board │ ├── knowledge-core/ # forge_knowledge diff --git a/RELEASE_READINESS.md b/RELEASE_READINESS.md index 0d4b1e09..e1ee6e25 100644 --- a/RELEASE_READINESS.md +++ b/RELEASE_READINESS.md @@ -2,8 +2,8 @@ - **Target bar:** Beta - **Overall verdict:** ❌ **NOT MET** -- **Generated (UTC):** 2026-07-10T16:14:12Z -- **Commit:** `34d48693fe97ca9c29845949573b0a9da29ea897` +- **Generated (UTC):** 2026-07-21T04:40:04Z +- **Commit:** `16974346f379a6e1cc2761ab90bef50b4decf910` - **Version (cz):** `0.1.0` > A bar is **MET** only when every gate at-or-below it is `GREEN` or `MANUAL_ATTESTED`. `SKIPPED_NO_CREDS`, `MISSING_EVIDENCE`, `STALE`, `MANUAL_PENDING`, and `RED` all mean **NOT MET** — the engine never infers a pass. @@ -12,16 +12,16 @@ | Gate | Blocker | Workstream | Status | Evidence (cmd/artifact) | Last-checked | |---|---|---|---|---|---| -| `G-DB` | #6 | HARD-01 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m postgres -q packages/db | 2026-07-10T16:14:12Z | -| `G-MODEL` | #1 | HARD-02 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q apps/api -k model_provider | 2026-07-10T16:14:12Z | -| `G-RAG-REAL` | #2 | HARD-04 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m realeval -q | 2026-07-10T16:14:12Z | -| `G-GH` | #1 | HARD-05 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q -k github_app | 2026-07-10T16:14:12Z | -| `G-MCP` | #1 | HARD-06 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q -k mcp_live | 2026-07-10T16:14:12Z | -| `G-SLACK` | #1 | HARD-07 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q -k slack_live | 2026-07-10T16:14:12Z | -| `G-BUILD` | #3 | HARD-08 | 🟢 GREEN | deploy/build-manifest.json | 2026-07-10T16:14:12Z | -| `G-TYPES` | #6 | HARD-12 | 🟢 GREEN | make typecheck | 2026-07-10T16:14:12Z | -| `G-SEC-AUTOMATED` | #4 | HARD-09 | 🟢 GREEN | uv run pytest -m security -q | 2026-07-10T16:14:12Z | -| `G-CRYPTO` | #5 | HARD-10 | 🟢 GREEN | uv run pytest -q apps/api/tests/test_auth_crypto_envelope.py apps/api/tests/test_cli_secrets.py | 2026-07-10T16:14:12Z | +| `G-DB` | #6 | HARD-01 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m postgres -q packages/db | 2026-07-21T04:40:04Z | +| `G-MODEL` | #1 | HARD-02 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q apps/api -k model_provider | 2026-07-21T04:40:04Z | +| `G-RAG-REAL` | #2 | HARD-04 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m realeval -q | 2026-07-21T04:40:04Z | +| `G-GH` | #1 | HARD-05 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q -k github_app | 2026-07-21T04:40:04Z | +| `G-MCP` | #1 | HARD-06 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q -k mcp_live | 2026-07-21T04:40:04Z | +| `G-SLACK` | #1 | HARD-07 | ⏭️ SKIPPED_NO_CREDS | uv run pytest -m integration -q -k slack_live | 2026-07-21T04:40:04Z | +| `G-BUILD` | #3 | HARD-08 | 🟢 GREEN | deploy/build-manifest.json | 2026-07-21T04:40:04Z | +| `G-TYPES` | #6 | HARD-12 | 🟢 GREEN | make typecheck | 2026-07-21T04:40:04Z | +| `G-SEC-AUTOMATED` | #4 | HARD-09 | 🟢 GREEN | uv run pytest -m security -q | 2026-07-21T04:40:04Z | +| `G-CRYPTO` | #5 | HARD-10 | 🟢 GREEN | uv run pytest -q apps/api/tests/test_auth_crypto_envelope.py apps/api/tests/test_cli_secrets.py | 2026-07-21T04:40:04Z | ## Verdict diff --git a/apps/api/forge_api/cli_verify.py b/apps/api/forge_api/cli_verify.py index ca4c40e0..4e0b3027 100644 --- a/apps/api/forge_api/cli_verify.py +++ b/apps/api/forge_api/cli_verify.py @@ -31,9 +31,11 @@ import argparse import base64 +import binascii import hashlib import sys import uuid +from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any @@ -49,7 +51,13 @@ if TYPE_CHECKING: from sqlalchemy.orm import Session, sessionmaker -__all__ = ["build_parser", "main"] +__all__ = [ + "StoredAttestationVerification", + "build_parser", + "main", + "resolve_verification_key", + "verify_stored_attestation", +] # --------------------------------------------------------------------------- # @@ -64,14 +72,15 @@ def _read_source(path: str) -> str: return Path(path).read_text(encoding="utf-8") -def _resolve_public_key(explicit: str | None) -> str: +def resolve_verification_key(explicit: str | None = None) -> str: """The base64 Ed25519 public key to verify against. - An explicit ``--public-key`` wins; otherwise fall back to the public half of - the env signing key (``FORGE_ATTEST_SIGNING_KEY``) so an attestation signed - on this host can be verified without re-passing the key. When no signing key - is set, ``EnvSigningKeyProvider`` warns and generates an ephemeral key whose - public half will simply not match a real signature — an honest REJECT. + An explicit key (``--public-key`` on the CLI) wins; otherwise fall back to + the public half of the env signing key (``FORGE_ATTEST_SIGNING_KEY``) so an + attestation signed on this host can be verified without re-passing the key. + When no signing key is set, ``EnvSigningKeyProvider`` warns and generates an + ephemeral key whose public half will simply not match a real signature — an + honest REJECT. """ if explicit: return explicit @@ -80,6 +89,62 @@ def _resolve_public_key(explicit: str | None) -> str: return EnvSigningKeyProvider().public_key_b64 +@dataclass(frozen=True) +class StoredAttestationVerification: + """Outcome of verifying a stored :class:`Attestation` row's envelope. + + ``payload_hash_ok`` — the sha256 over the envelope's PAE re-derivation + matches the recorded ``payload_hash`` column; ``signature_ok`` — an Ed25519 + signature on the envelope verifies for the resolved public key. ``error`` + is set (and both flags are ``False``) when the envelope payload is not + valid base64 — a malformed record is "not verified", never an exception. + """ + + payload_hash_ok: bool + signature_ok: bool + recomputed_payload_hash: str | None = None + error: str | None = None + + @property + def ok(self) -> bool: + """True iff the recorded hash matches AND the signature verifies.""" + return self.payload_hash_ok and self.signature_ok + + +def verify_stored_attestation( + envelope: DsseEnvelope, + recorded_payload_hash: str, + *, + public_key_b64: str | None = None, +) -> StoredAttestationVerification: + """Verify a stored attestation exactly as ``forge-verify --run`` does. + + Re-derives ``payload_hash`` from the envelope's PAE encoding, confirms it + matches the recorded column, and Ed25519-verifies the signature against + ``public_key_b64`` (resolved via :func:`resolve_verification_key` when not + given). This is the single DB-backed verification seam — the REST surface + (``forge_api.routers.attestations``) and the CLI's ``--run`` mode both call + it, so "verified" can never mean two different things. + """ + try: + payload_bytes = base64.b64decode(envelope.payload, validate=True) + except (binascii.Error, ValueError, TypeError) as exc: + return StoredAttestationVerification( + payload_hash_ok=False, + signature_ok=False, + error=f"envelope payload is not valid base64: {exc}", + ) + recomputed = hashlib.sha256(pae(envelope.payloadType, payload_bytes)).hexdigest() + signature_ok = DsseVerifier().verify( + envelope, public_key_b64=resolve_verification_key(public_key_b64) + ) + return StoredAttestationVerification( + payload_hash_ok=recomputed == recorded_payload_hash, + signature_ok=signature_ok, + recomputed_payload_hash=recomputed, + ) + + def _print_envelope_summary(envelope: DsseEnvelope) -> None: """Print the envelope's predicate type + first subject digest, best-effort.""" print(f"payloadType: {envelope.payloadType}") @@ -129,7 +194,7 @@ def _cmd_attestation(args: argparse.Namespace) -> int: except (OSError, ValueError) as exc: print(f"error: invalid DSSE envelope: {exc}", file=sys.stderr) return 1 - public_key = _resolve_public_key(args.public_key) + public_key = resolve_verification_key(args.public_key) ok = DsseVerifier().verify(envelope, public_key_b64=public_key) _print_envelope_summary(envelope) print("VERIFIED" if ok else "REJECTED") @@ -174,24 +239,20 @@ def _cmd_run(args: argparse.Namespace) -> int: keyid = row.keyid attestation_id = row.id - try: - payload_bytes = base64.b64decode(envelope.payload, validate=True) - except (ValueError, TypeError) as exc: - print(f"error: envelope payload is not valid base64: {exc}", file=sys.stderr) + result = verify_stored_attestation(envelope, recorded_hash, public_key_b64=args.public_key) + if result.error is not None: + print(f"error: {result.error}", file=sys.stderr) return 1 - recomputed_hash = hashlib.sha256(pae(envelope.payloadType, payload_bytes)).hexdigest() - hash_ok = recomputed_hash == recorded_hash - - public_key = _resolve_public_key(args.public_key) - sig_ok = DsseVerifier().verify(envelope, public_key_b64=public_key) print(f"attestation: {attestation_id} (keyid {keyid})") _print_envelope_summary(envelope) - print(f"payload_hash match: {hash_ok} (recorded={recorded_hash}, recomputed={recomputed_hash})") - print(f"signature verified: {sig_ok}") - ok = hash_ok and sig_ok - print("VERIFIED" if ok else "REJECTED") - return 0 if ok else 1 + print( + f"payload_hash match: {result.payload_hash_ok} " + f"(recorded={recorded_hash}, recomputed={result.recomputed_payload_hash})" + ) + print(f"signature verified: {result.signature_ok}") + print("VERIFIED" if result.ok else "REJECTED") + return 0 if result.ok else 1 # --------------------------------------------------------------------------- # diff --git a/apps/api/forge_api/routers/__init__.py b/apps/api/forge_api/routers/__init__.py index 0fd847b0..e3b381f4 100644 --- a/apps/api/forge_api/routers/__init__.py +++ b/apps/api/forge_api/routers/__init__.py @@ -15,8 +15,8 @@ agent, alerts, ao_settings, - approval, approvals, + attestations, audit, auth, automations, @@ -64,8 +64,8 @@ marketplace.router, benchmarks.router, integration.router, - approval.router, approvals.router, + attestations.router, audit.router, cost.router, incidents.router, @@ -91,8 +91,8 @@ "agent", "alerts", "ao_settings", - "approval", "approvals", + "attestations", "audit", "auth", "automations", diff --git a/apps/api/forge_api/routers/agent.py b/apps/api/forge_api/routers/agent.py index 1c502724..e496ad2c 100644 --- a/apps/api/forge_api/routers/agent.py +++ b/apps/api/forge_api/routers/agent.py @@ -105,6 +105,19 @@ def get(self, run_id: uuid.UUID, *, workspace_id: uuid.UUID) -> AgentRunResult | return None return self._runs.get(run_id) + def owner_of(self, run_id: uuid.UUID) -> uuid.UUID | None: + """Return the workspace that owns ``run_id``, or ``None`` if unknown. + + Read-only, cross-tenant-safe resolution mirroring + :meth:`~forge_api.routers.integration.ApprovalStore.owner_of`: the Slack + ``/forge status`` slash command + is unauthenticated untrusted intake (it carries no Forge principal), so + the handler resolves the owning workspace from the run id and then reads + the run back through the normal workspace-scoped :meth:`get`. An unknown + id yields ``None`` -> the handler returns its not-found copy. + """ + return self._owner.get(run_id) + # --------------------------------------------------------------------------- # # Runner + store dependencies (overridable for tests / BYOK swap) # diff --git a/apps/api/forge_api/routers/ao_settings.py b/apps/api/forge_api/routers/ao_settings.py index dabc7bef..9b59ee7c 100644 --- a/apps/api/forge_api/routers/ao_settings.py +++ b/apps/api/forge_api/routers/ao_settings.py @@ -11,6 +11,10 @@ * ``PUT /ao/settings`` — partially update those (admin). * ``POST /ao/routing-preview`` — what tier/model/strategy a sample task would get, given this workspace's current settings. +* ``GET /ao/self-eval/status`` — the Self-Eval Gate facts: enforcement + flag, the workspace's private suite, and the recorded baseline. +* ``POST /ao/self-eval/runs`` — enqueue the worker-owned + ``forge.self_eval.run`` task for this workspace (admin, 202). Reads are ``Permission.READ``-gated (a viewer may inspect its own workspace's AO configuration); every mutation is ``Permission.ADMIN``-gated, matching the @@ -24,6 +28,8 @@ from typing import Annotated, Any from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy import select +from sqlalchemy.orm import Session from forge_api.auth.rbac import Permission from forge_api.deps import DbSession, Principal, get_current_principal @@ -36,7 +42,12 @@ RoleConfigUpsertRequest, RoutingPreviewRequest, RoutingPreviewResponse, + SelfEvalBaselineOut, + SelfEvalRunAccepted, + SelfEvalStatusOut, + SelfEvalSuiteOut, ) +from forge_api.services import self_eval_service from forge_api.services.ao_settings_service import ( AoSettingsService, EffectiveAoSettings, @@ -47,6 +58,7 @@ from forge_contracts.audit import AuditEvent from forge_contracts.orchestration_config import AgentRole, EffectiveRoleConfig from forge_db.ao_settings import SqlAoSettingsStore +from forge_db.models.benchmark import BenchmarkSuite, SelfEvalBaseline from forge_db.role_config import SqlRoleConfigStore from forge_eval.sweval import SelfEvalGate, SelfEvalRegressionError @@ -315,3 +327,118 @@ def routing_preview( medior_max=preview.medior_max, auto_route_enabled=preview.auto_route_enabled, ) + + +# --------------------------------------------------------------------------- # +# Self-Eval Gate: status read + run trigger (Phase A) # +# --------------------------------------------------------------------------- # + + +def _private_suite( + session: Session, workspace_id: uuid.UUID, *, published_only: bool = False +) -> BenchmarkSuite | None: + """The workspace's private Self-Eval suite (published preferred), or ``None``.""" + stmt = select(BenchmarkSuite).where( + BenchmarkSuite.workspace_id == workspace_id, + BenchmarkSuite.private.is_(True), + ) + if published_only: + stmt = stmt.where(BenchmarkSuite.published.is_(True)) + stmt = stmt.order_by(BenchmarkSuite.published.desc(), BenchmarkSuite.version.desc()) + return session.scalars(stmt).first() + + +def _effective_config_snapshot(session: Session, workspace_id: uuid.UUID) -> dict[str, Any]: + """The redacted effective AO settings a self-eval run scores (no secrets).""" + effective = _service(session).get_settings(workspace_id) + return { + "scope": "ao.settings", + "auto_route": effective.auto_route, + "tier_model_overrides": effective.tier_model_overrides, + "junior_max": effective.junior_max, + "medior_max": effective.medior_max, + } + + +@router.get( + "/self-eval/status", + summary="Self-Eval Gate facts: enforcement flag, private suite, recorded baseline.", +) +def self_eval_status(principal: ReaderDep, session: DbSession) -> SelfEvalStatusOut: + suite = _private_suite(session, principal.workspace_id) + baseline = session.scalars( + select(SelfEvalBaseline) + .where(SelfEvalBaseline.workspace_id == principal.workspace_id) + .order_by(SelfEvalBaseline.updated_at.desc()) + ).first() + return SelfEvalStatusOut( + workspace_id=principal.workspace_id, + enforced=get_app_settings().self_eval_enforce, + suite=SelfEvalSuiteOut( + id=suite.id, + slug=suite.slug, + version=suite.version, + title=suite.title, + task_count=suite.task_count, + repo_id=suite.repo_id, + published=suite.published, + ) + if suite is not None + else None, + baseline=SelfEvalBaselineOut( + benchmark_suite_id=baseline.benchmark_suite_id, + baseline_rate=baseline.baseline_rate, + resolved=baseline.resolved, + total=baseline.total, + recorded_at=baseline.updated_at, + ) + if baseline is not None + else None, + ) + + +@router.post( + "/self-eval/runs", + status_code=status.HTTP_202_ACCEPTED, + summary="Enqueue the worker-owned self-eval run for this workspace (admin).", +) +def request_self_eval_run(principal: AdminDep, session: DbSession) -> SelfEvalRunAccepted: + """Queue ``forge.self_eval.run`` over the workspace's private suite. + + The run itself stays in the worker (minutes-long, agent-driven, A4) — this + endpoint only enqueues it with the workspace's current effective AO config + snapshot. 409 when no published private suite exists: there is nothing the + worker could score, so we refuse rather than queue a guaranteed no-op. + """ + suite = _private_suite(session, principal.workspace_id, published_only=True) + if suite is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "error": "no_private_suite", + "message": ( + "No published private Self-Eval suite exists for this " + "workspace; one is minted from merged PRs by the " + "forge.self_eval.mint worker task." + ), + }, + ) + self_eval_service.enqueue_self_eval_run( + principal.workspace_id, + _effective_config_snapshot(session, principal.workspace_id), + recorded_by=principal.user_id, + ) + _audit( + session, + principal, + "ao.self_eval.run_requested", + result="success", + severity="info", + details={"benchmark_suite_id": str(suite.id)}, + ) + session.commit() + return SelfEvalRunAccepted( + task=self_eval_service.SELF_EVAL_RUN_TASK, + workspace_id=principal.workspace_id, + benchmark_suite_id=suite.id, + ) diff --git a/apps/api/forge_api/routers/approval.py b/apps/api/forge_api/routers/approval.py deleted file mode 100644 index 000899d6..00000000 --- a/apps/api/forge_api/routers/approval.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Approval router — the human-in-the-loop gate surface (wired in Phase 2 Task 2.1). - -Serves the approval queue over HTTP: - -* ``POST /approval/requests`` — open an approval request (a gate - raised by the workflow/agent layer when a task needs human sign-off). -* ``GET /approval/requests`` — list pending/decided requests. -* ``GET /approval/requests/{approval_id}`` — fetch one request (full context). -* ``POST /approval/requests/{approval_id}/decision`` — approve / reject / - request changes. - -Handlers delegate to a process-wide in-memory :class:`ApprovalStore` (the -DB-backed store is swapped in behind the same dependency via -``app.dependency_overrides`` / config). Unknown ids map to HTTP 404. -""" - -from __future__ import annotations - -import uuid -from datetime import UTC, datetime -from functools import lru_cache -from typing import Annotated - -from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic import BaseModel - -from forge_api.auth.rbac import Permission -from forge_api.deps import Principal, get_current_principal -from forge_api.routers._rbac import require_permission -from forge_contracts import ApprovalRequest -from forge_contracts.enums import ApprovalStatus - -router = APIRouter( - prefix="/approval", - tags=["approval"], - dependencies=[Depends(get_current_principal)], -) - -# Permission-gated principals (authenticate + authorize, returning the principal -# so handlers can scope by workspace and record the decider identity). Opening -# and deciding a gate are WRITE operations — a read-only ``viewer`` and the -# ``agent-runner`` (which lacks WRITE) are therefore denied, preserving the -# human-in-the-loop guarantee that an agent cannot approve its own gate. -ReaderDep = Annotated[Principal, Depends(require_permission(Permission.READ))] -WriterDep = Annotated[Principal, Depends(require_permission(Permission.WRITE))] - - -# --------------------------------------------------------------------------- # -# In-memory approval store # -# --------------------------------------------------------------------------- # - - -class ApprovalStore: - """A tiny in-memory store of approval requests (keyed by id). - - Each request is tagged with the ``workspace_id`` that created it; every read - and decision is scoped to a workspace so one tenant can never see, fetch, or - decide another tenant's gates. ``ApprovalRequest`` carries no ``workspace_id`` - field (it is a frozen contract), so ownership is tracked alongside the items. - """ - - def __init__(self) -> None: - self._items: dict[uuid.UUID, ApprovalRequest] = {} - self._owner: dict[uuid.UUID, uuid.UUID] = {} - - def create(self, request: ApprovalRequest, *, workspace_id: uuid.UUID) -> ApprovalRequest: - if request.id is None: - request.id = uuid.uuid4() - if request.created_at is None: - request.created_at = datetime.now(UTC) - self._items[request.id] = request - self._owner[request.id] = workspace_id - return request - - def list( - self, *, workspace_id: uuid.UUID, status: ApprovalStatus | None = None - ) -> list[ApprovalRequest]: - items = [req for key, req in self._items.items() if self._owner.get(key) == workspace_id] - if status is not None: - items = [i for i in items if i.status == status] - return items - - def get(self, approval_id: uuid.UUID, *, workspace_id: uuid.UUID) -> ApprovalRequest | None: - if self._owner.get(approval_id) != workspace_id: - return None - return self._items.get(approval_id) - - def owner_of(self, approval_id: uuid.UUID) -> uuid.UUID | None: - """Return the workspace that owns ``approval_id``, or ``None`` if unknown. - - Read-only, cross-tenant-safe resolution used by the Slack interactivity - handler: a Slack ``block_actions`` callback is unauthenticated (it carries - no Forge principal) and embeds only the approval id, so the handler must - resolve the owning workspace before applying the decision through the - normal workspace-scoped :meth:`decide` path. An unknown id yields - ``None`` -> the handler no-ops (never leaks another tenant's gate). - """ - return self._owner.get(approval_id) - - def decide( - self, - approval_id: uuid.UUID, - *, - workspace_id: uuid.UUID, - status: ApprovalStatus, - decided_by: str | None, - reason: str | None, - ) -> ApprovalRequest | None: - request = self.get(approval_id, workspace_id=workspace_id) - if request is None: - return None - request.status = status - request.decided_by = decided_by - request.decision_reason = reason - request.decided_at = datetime.now(UTC) - return request - - -@lru_cache(maxsize=1) -def _approval_store_singleton() -> ApprovalStore: - return ApprovalStore() - - -def get_approval_store() -> ApprovalStore: - """Return the process-wide approval store (override in tests via DI).""" - return _approval_store_singleton() - - -StoreDep = Annotated[ApprovalStore, Depends(get_approval_store)] - - -# --------------------------------------------------------------------------- # -# Request bodies # -# --------------------------------------------------------------------------- # - - -class DecisionRequest(BaseModel): - """Body for ``POST /approval/requests/{approval_id}/decision``. - - There is deliberately **no** ``decided_by`` field: the decider identity is - taken from the authenticated principal, never from the request body, so the - gate's accountability record cannot be forged. - """ - - status: ApprovalStatus - reason: str | None = None - - -def _decider_identity(principal: Principal) -> str: - """Stable, human-readable identity for the authenticated decider.""" - return principal.email or str(principal.user_id) - - -def _require(request: ApprovalRequest | None, approval_id: uuid.UUID) -> ApprovalRequest: - if request is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail=f"no approval request {approval_id}" - ) - return request - - -# --------------------------------------------------------------------------- # -# Routes # -# --------------------------------------------------------------------------- # - - -@router.post("/requests", response_model=ApprovalRequest, status_code=status.HTTP_201_CREATED) -def create_request( - store: StoreDep, principal: WriterDep, request: ApprovalRequest -) -> ApprovalRequest: - """Open an approval request in the caller's workspace.""" - return store.create(request, workspace_id=principal.workspace_id) - - -@router.get("/requests", response_model=list[ApprovalRequest]) -def list_requests( - store: StoreDep, - principal: ReaderDep, - status: Annotated[ApprovalStatus | None, Query()] = None, -) -> list[ApprovalRequest]: - """List the caller workspace's approval requests (optionally by status).""" - return store.list(workspace_id=principal.workspace_id, status=status) - - -@router.get("/requests/{approval_id}", response_model=ApprovalRequest) -def get_request(store: StoreDep, principal: ReaderDep, approval_id: uuid.UUID) -> ApprovalRequest: - """Fetch one approval request (only within the caller's workspace).""" - return _require(store.get(approval_id, workspace_id=principal.workspace_id), approval_id) - - -@router.post("/requests/{approval_id}/decision", response_model=ApprovalRequest) -def decide( - store: StoreDep, - principal: WriterDep, - approval_id: uuid.UUID, - payload: DecisionRequest, -) -> ApprovalRequest: - """Approve / reject / request changes on an approval request. - - The decider identity is the authenticated principal (WRITE-capable: a human - ``member``/``admin``); the read-only ``viewer`` and the ``agent-runner`` are - rejected upstream by the WRITE gate, so an agent cannot decide its own gate. - """ - decided = store.decide( - approval_id, - workspace_id=principal.workspace_id, - status=payload.status, - decided_by=_decider_identity(principal), - reason=payload.reason, - ) - return _require(decided, approval_id) - - -__all__ = ["ApprovalStore", "DecisionRequest", "get_approval_store", "router"] diff --git a/apps/api/forge_api/routers/attestations.py b/apps/api/forge_api/routers/attestations.py new file mode 100644 index 00000000..ad1b1658 --- /dev/null +++ b/apps/api/forge_api/routers/attestations.py @@ -0,0 +1,166 @@ +"""Attested Changesets read-only REST surface (Task 19). + +* ``GET /attestations`` — workspace-scoped page, newest first +* ``GET /attestations/{id}`` — one record (foreign ids look nonexistent) +* ``GET /approvals/{id}/attestation`` — the record minted when the gate's + linked workflow run was attested (404 when none) + +Read-only by design: attestations are minted exclusively as a side effect of +approving a ``pr`` gate (``PrAttestationResolutionHook``); there is no HTTP +route that creates one, mirroring the audit router's producer/consumer split. +Reads go straight through the append-only ``AttestationRepository`` scoped to +the caller's workspace on the row itself (the red-team surface's convention). + +``verified`` is computed per record by the exact seam ``forge-verify --run`` +uses (:func:`forge_api.cli_verify.verify_stored_attestation`): re-derive +``payload_hash`` from the envelope's PAE and Ed25519-verify against the +deployment's verification key (the public half of ``FORGE_ATTEST_SIGNING_KEY``, +resolved once per request). Signature verification is never re-implemented +here, so the REST answer and the CLI's exit code can never disagree. + +Auth/tenancy mirror the F36 approvals router exactly: every route hangs off the +authenticated principal (READ permission) and scopes by +``principal.workspace_id``; cross-workspace ids map to ``404`` (no existence +leak). +""" + +from __future__ import annotations + +import uuid +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy.orm import Session + +from forge_api.auth.rbac import Permission +from forge_api.cli_verify import resolve_verification_key, verify_stored_attestation +from forge_api.db import get_db +from forge_api.deps import Principal, get_current_principal +from forge_api.routers._rbac import require_permission +from forge_api.schemas.attestation import ( + AttestationListResponse, + AttestationOut, + AttestationProvenance, +) +from forge_api.services.approval_service import get_approval_service +from forge_approval import ApprovalNotFoundError, ApprovalService +from forge_contracts.attestation import DsseEnvelope +from forge_db.attest.repository import AttestationRepository +from forge_db.models import Attestation + +router = APIRouter(tags=["attestations"], dependencies=[Depends(get_current_principal)]) + +ReaderDep = Annotated[Principal, Depends(require_permission(Permission.READ))] +SessionDep = Annotated[Session, Depends(get_db)] +ApprovalServiceDep = Annotated[ApprovalService, Depends(get_approval_service)] + + +def _to_out(row: Attestation, *, public_key_b64: str) -> AttestationOut: + """Map one ORM row onto the response schema, verifying it live. + + Verification is total (a malformed envelope is "not verified", never a + 500): both the envelope re-validation and the shared seam degrade to + ``verified=False`` on any decode failure. + """ + try: + envelope = DsseEnvelope.model_validate(row.envelope) + except ValueError: + verified = False + else: + verified = verify_stored_attestation( + envelope, row.payload_hash, public_key_b64=public_key_b64 + ).ok + return AttestationOut( + id=row.id, + changeset_hash=row.subject_digest, + predicate_type=row.predicate_type, + keyid=row.keyid, + payload_hash=row.payload_hash, + created_at=row.created_at, + verified=verified, + provenance=AttestationProvenance.model_validate(row), + ) + + +def _not_found(attestation_id: uuid.UUID) -> HTTPException: + return HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"no attestation {attestation_id}" + ) + + +@router.get("/attestations", response_model=AttestationListResponse) +def list_attestations( + principal: ReaderDep, + session: SessionDep, + workflow_run_id: Annotated[uuid.UUID | None, Query()] = None, + agent_run_id: Annotated[uuid.UUID | None, Query()] = None, + spec_key: Annotated[str | None, Query()] = None, + limit: Annotated[int, Query(ge=1, le=200)] = 50, + offset: Annotated[int, Query(ge=0)] = 0, +) -> AttestationListResponse: + """One workspace-scoped page of attestations, newest first.""" + rows = AttestationRepository(session).list( + principal.workspace_id, + workflow_run_id=workflow_run_id, + agent_run_id=agent_run_id, + spec_key=spec_key, + limit=limit, + offset=offset, + ) + public_key = resolve_verification_key() + return AttestationListResponse( + items=[_to_out(row, public_key_b64=public_key) for row in rows], + limit=limit, + offset=offset, + ) + + +@router.get("/attestations/{attestation_id}", response_model=AttestationOut) +def get_attestation( + principal: ReaderDep, session: SessionDep, attestation_id: uuid.UUID +) -> AttestationOut: + """One attestation (workspace-scoped; cross-workspace ids look nonexistent).""" + row = session.get(Attestation, attestation_id) + if row is None or row.workspace_id != principal.workspace_id: + raise _not_found(attestation_id) + return _to_out(row, public_key_b64=resolve_verification_key()) + + +@router.get("/approvals/{approval_id}/attestation", response_model=AttestationOut) +async def get_approval_attestation( + principal: ReaderDep, + session: SessionDep, + service: ApprovalServiceDep, + approval_id: uuid.UUID, +) -> AttestationOut: + """The attestation minted for the gate's linked workflow run (404 when none). + + Resolves the gate through the same workspace-scoped ``ApprovalService.get`` + the approvals router uses (foreign gate ids 404 identically), then reads the + newest attestation for its ``workflow_run_id``. A gate without a linked run, + or whose run was never attested (e.g. still pending), is an honest 404 — + the UI renders that as "not attested", never a fake state. + """ + try: + request = await service.get(approval_id, workspace_id=principal.workspace_id) + except ApprovalNotFoundError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f"no approval request {approval_id}" + ) from None + + row = ( + AttestationRepository(session).get_by_run( + principal.workspace_id, workflow_run_id=request.workflow_run_id + ) + if request.workflow_run_id is not None + else None + ) + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"no attestation recorded for approval {approval_id}", + ) + return _to_out(row, public_key_b64=resolve_verification_key()) + + +__all__ = ["router"] diff --git a/apps/api/forge_api/routers/integration.py b/apps/api/forge_api/routers/integration.py index d3b7226c..71e23ba5 100644 --- a/apps/api/forge_api/routers/integration.py +++ b/apps/api/forge_api/routers/integration.py @@ -19,6 +19,7 @@ import contextlib import uuid +from datetime import UTC, datetime from functools import lru_cache from typing import Annotated from urllib.parse import parse_qs @@ -32,9 +33,10 @@ from forge_api.observability.audit import AuditCategory, AuditLog from forge_api.observability.redaction import redact_mapping, redact_text from forge_api.routers._rbac import require_permission -from forge_api.routers.approval import ApprovalStore, get_approval_store +from forge_api.routers.agent import AgentRunStore, get_agent_store from forge_api.settings import Settings, get_settings from forge_contracts import ( + ApprovalRequest, CIStatus, HealthResult, PullRequest, @@ -202,15 +204,117 @@ def _slack_notifier_singleton() -> SlackNotifier: ) +class ApprovalStore: + """In-memory approval queue that backs **only** the Slack integration. + + This is the Phase-2 in-memory approval store. Its HTTP surface (the legacy + ``/approval/*`` router) was retired in favour of the DB-backed F36 + ``/approvals/*`` surface (``routers/approvals.py`` + + ``forge_approval.ApprovalService``). The store itself survives here because + the Slack integration is still wired to it and **cannot** be migrated to the + F36 service within a bounded diff: + + * a Slack ``block_actions`` / ``chat.update`` callback is *unauthenticated* + untrusted intake — it carries no Forge principal, only an approval id — so + it needs :meth:`owner_of` to resolve the owning workspace from the id + alone; ``ApprovalService`` deliberately exposes no such id-only lookup + (every read demands a ``workspace_id``, and a cross-workspace id maps to + not-found); + * ``ApprovalService.resolve`` requires a domain ``Principal`` of + ``kind="user"`` with a real Forge user UUID and a ``member``/``admin`` + role, and records that UUID as the decision's ``approver_user_id``; a Slack + actor is a ``slack:`` string with no Forge-user mapping anywhere in + the codebase, so it cannot satisfy the authorizer without forging identity; + * :meth:`slack_notify_approval` hands the resolved request to + :meth:`SlackNotifier.notify_approval`, which is typed to the frozen + :class:`~forge_contracts.ApprovalRequest` contract — a different type than + the F36 ``forge_approval.models.ApprovalRequest`` the service returns. + + Consumers (both in this module): :func:`slack_notify_approval` + (``POST /integration/slack/approvals/{id}/notify``) reads via :meth:`get`; + :func:`slack_interaction` (``POST /integration/slack/interactions``) resolves + via :meth:`owner_of` then applies :meth:`decide`. + + Each request is tagged with the ``workspace_id`` that created it; every read + and decision is scoped to a workspace so one tenant can never see, fetch, or + decide another tenant's gates. ``ApprovalRequest`` carries no ``workspace_id`` + field (it is a frozen contract), so ownership is tracked alongside the items. + + Note: since the legacy ``POST /approval/requests`` route was removed, nothing + populates this store in production — the Slack notify/decide flow below + operates on an empty store until a writer is reintroduced (tests seed it + directly). Kept pending migration of Slack interactivity to the DB-backed + ``/approvals`` surface. + """ + + def __init__(self) -> None: + self._items: dict[uuid.UUID, ApprovalRequest] = {} + self._owner: dict[uuid.UUID, uuid.UUID] = {} + + def create(self, request: ApprovalRequest, *, workspace_id: uuid.UUID) -> ApprovalRequest: + if request.id is None: + request.id = uuid.uuid4() + if request.created_at is None: + request.created_at = datetime.now(UTC) + self._items[request.id] = request + self._owner[request.id] = workspace_id + return request + + def get(self, approval_id: uuid.UUID, *, workspace_id: uuid.UUID) -> ApprovalRequest | None: + if self._owner.get(approval_id) != workspace_id: + return None + return self._items.get(approval_id) + + def owner_of(self, approval_id: uuid.UUID) -> uuid.UUID | None: + """Return the workspace that owns ``approval_id``, or ``None`` if unknown. + + Read-only, cross-tenant-safe resolution used by the Slack interactivity + handler: a Slack ``block_actions`` callback is unauthenticated (it carries + no Forge principal) and embeds only the approval id, so the handler must + resolve the owning workspace before applying the decision through the + normal workspace-scoped :meth:`decide` path. An unknown id yields + ``None`` -> the handler no-ops (never leaks another tenant's gate). + """ + return self._owner.get(approval_id) + + def decide( + self, + approval_id: uuid.UUID, + *, + workspace_id: uuid.UUID, + status: ApprovalStatus, + decided_by: str | None, + reason: str | None, + ) -> ApprovalRequest | None: + request = self.get(approval_id, workspace_id=workspace_id) + if request is None: + return None + request.status = status + request.decided_by = decided_by + request.decision_reason = reason + request.decided_at = datetime.now(UTC) + return request + + +@lru_cache(maxsize=1) +def _approval_store_singleton() -> ApprovalStore: + return ApprovalStore() + + +def get_approval_store() -> ApprovalStore: + """Return the process-wide Slack-integration approval store (override in tests).""" + return _approval_store_singleton() + + class SlackApprovalRefStore: """In-memory ``approval_id -> (channel, ts)`` back-reference for Slack posts. ``ApprovalRequest`` is a frozen contract (no ``slack_ts`` field), so — exactly - as :class:`~forge_api.routers.approval.ApprovalStore` tracks workspace - ownership *beside* the item rather than mutating the DTO — the resolved - channel + message timestamp of an approval's Slack post are tracked here. The - interactivity handler reads them to edit the original message in place - (``chat.update``). The DB-backed store swaps in behind the same dependency. + as :class:`ApprovalStore` tracks workspace ownership *beside* the item rather + than mutating the DTO — the resolved channel + message timestamp of an + approval's Slack post are tracked here. The interactivity handler reads them + to edit the original message in place (``chat.update``). The DB-backed store + swaps in behind the same dependency. """ def __init__(self) -> None: @@ -280,6 +384,7 @@ def get_slack_signing_secret() -> str | None: SlackSigningSecretDep = Annotated[str | None, Depends(get_slack_signing_secret)] ApprovalStoreDep = Annotated[ApprovalStore, Depends(get_approval_store)] SlackRefStoreDep = Annotated[SlackApprovalRefStore, Depends(get_slack_ref_store)] +AgentRunStoreDep = Annotated[AgentRunStore, Depends(get_agent_store)] # --------------------------------------------------------------------------- # @@ -488,17 +593,43 @@ def _slack_actor(payload: dict[str, object]) -> str: return "slack:unknown" -def _command_blocks(text: str) -> dict[str, object]: +def _run_status_body(store: AgentRunStore, raw_run_id: str) -> str: + """Resolve ``/forge status `` against the live agent-run store. + + Mirrors the interactivity handler's ``owner_of`` pattern: the slash command is + unauthenticated untrusted intake (no Forge principal), so the owning workspace + is resolved from the run id and the run is then read back through the normal + workspace-scoped :meth:`AgentRunStore.get`. A malformed or unknown id yields + the not-found copy (never fabricated). The response reports the run's real + :class:`~forge_contracts.enums.RunStatus`, plus the number of recorded steps + when the run carries any (the only step/phase detail the run model holds). + """ + try: + run_id = uuid.UUID(raw_run_id) + except ValueError: + return f"*Forge* — no run `{raw_run_id}` found." + owner = store.owner_of(run_id) + result = store.get(run_id, workspace_id=owner) if owner is not None else None + if result is None: + return f"*Forge* — no run `{raw_run_id}` found." + detail = "" + if result.steps: + count = len(result.steps) + detail = f" ({count} step{'s' if count != 1 else ''})" + return f"run {run_id}: {result.status.value}{detail}" + + +def _command_blocks(text: str, store: AgentRunStore) -> dict[str, object]: """Build the ephemeral Block Kit response for a ``/forge`` sub-command.""" parts = text.strip().split() sub = parts[0].lower() if parts else "help" if sub == "status" and len(parts) > 1: - body = f"*Forge* — status for `{parts[1]}` is not wired to a live task yet." + body = _run_status_body(store, parts[1]) else: body = ( "*Forge* commands:\n" "• `/forge help` — show this help\n" - "• `/forge status ` — task status" + "• `/forge status ` — live run status" ) return { "response_type": "ephemeral", @@ -511,20 +642,23 @@ async def slack_slash_command( request: Request, secret: SlackSigningSecretDep, settings: SettingsDep, + store: AgentRunStoreDep, x_slack_signature: Annotated[str | None, Header()] = None, x_slack_request_timestamp: Annotated[str | None, Header()] = None, ) -> JSONResponse: """Handle a signed ``/forge`` slash command (``x-www-form-urlencoded``). Verifies the Slack v0 signature over the raw body, then returns an ephemeral - Block Kit response within Slack's 3-second budget. Fail-closed: 501 when no - signing secret is configured, 401 on a bad/missing/stale signature. + Block Kit response within Slack's 3-second budget. ``/forge status `` is + resolved live against the shared :class:`~forge_api.routers.agent.AgentRunStore` + (the same store the runs API serves ``GET /agent/runs/{id}`` from). Fail-closed: + 501 when no signing secret is configured, 401 on a bad/missing/stale signature. """ body = await request.body() _verify_slack_request(secret, settings, body, x_slack_request_timestamp, x_slack_signature) form = parse_qs(body.decode("utf-8", errors="replace")) text = (form.get("text") or [""])[0] - return JSONResponse(content=_command_blocks(text)) + return JSONResponse(content=_command_blocks(text, store)) @router.post("/slack/interactions") @@ -630,8 +764,10 @@ async def slack_interaction( __all__ = [ + "ApprovalStore", "RepoConnectionStore", "SlackApprovalRefStore", + "get_approval_store", "get_github_client", "get_github_client_optional", "get_github_webhook_secret", diff --git a/apps/api/forge_api/routers/pm.py b/apps/api/forge_api/routers/pm.py index ae26d640..81780b9e 100644 --- a/apps/api/forge_api/routers/pm.py +++ b/apps/api/forge_api/routers/pm.py @@ -4,10 +4,14 @@ signature/secret-verified webhook intake routes (no bearer). All queries are workspace-scoped (cross-workspace ids -> 404, no existence leak). -Parked (needs the F01 Postgres board substrate — see ``pm_service`` docstring and -the slice report): OAuth code-exchange routes, ``backfill`` enqueue, the manual -conflict ``resolve`` execution, and the worker board-write/scan tasks. The -sync engine that performs those is fully unit-tested in ``forge_integrations``. +An accepted webhook now completes the inbound loop: the service persists the +delivery and enqueues the worker board-write task ``forge.pm.process_webhook`` +(``forge_worker.tasks.pm_sync`` — re-fetch through the provider adapter, then +``PMSyncEngine.sync_in`` onto the F01 Postgres board substrate, workspace-scoped +and idempotent on redelivery; see the ``pm_service`` docstring). + +Still parked: OAuth code-exchange routes, ``backfill`` enqueue, the manual +conflict ``resolve`` execution, and the outbound ``activity_events`` scan. """ from __future__ import annotations diff --git a/apps/api/forge_api/routers/spec.py b/apps/api/forge_api/routers/spec.py index 56533678..8ef0c555 100644 --- a/apps/api/forge_api/routers/spec.py +++ b/apps/api/forge_api/routers/spec.py @@ -178,6 +178,12 @@ class TextContent(BaseModel): content: str +class SpecReviewRequest(BaseModel): + """Body for the reject / request-changes review endpoints.""" + + note: str = "" + + class DraftSpecRequest(BaseModel): """Body for ``POST /spec/draft`` (BYOK AI spec drafting; draft-only).""" @@ -343,6 +349,32 @@ def approve_spec(engine: EngineDep, spec_id: uuid.UUID) -> SpecManifest: return engine.approve_spec(spec_id) +@router.post("/specs/{spec_id}/reject", response_model=SpecManifest, dependencies=[WriteGate]) +def reject_spec(engine: EngineDep, spec_id: uuid.UUID, request: SpecReviewRequest) -> SpecManifest: + """Reject a spec at the human gate; moves it to ``rejected``. + + Persists the reviewer's note in the manifest. 409 when the spec is already + past the gate (``approved`` and beyond), mirroring the tasks gate. + """ + with _spec_errors(): + return engine.reject_spec(spec_id, request.note) + + +@router.post( + "/specs/{spec_id}/request-changes", response_model=SpecManifest, dependencies=[WriteGate] +) +def request_changes( + engine: EngineDep, spec_id: uuid.UUID, request: SpecReviewRequest +) -> SpecManifest: + """Request changes on a spec at the human gate; moves it to ``changes_requested``. + + The counterpart of the reject endpoint for the softer review outcome; same + persistence and 409 gating semantics. + """ + with _spec_errors(): + return engine.request_changes(spec_id, request.note) + + @router.post("/specs/{spec_id}/tasks", response_model=list[TaskDTO], dependencies=[WriteGate]) def spec_tasks(engine: EngineDep, spec_id: uuid.UUID) -> list[TaskDTO]: """Generate implementation tasks from an approved spec (gated).""" diff --git a/apps/api/forge_api/routers/workflow.py b/apps/api/forge_api/routers/workflow.py index 0872e019..de7f574d 100644 --- a/apps/api/forge_api/routers/workflow.py +++ b/apps/api/forge_api/routers/workflow.py @@ -6,19 +6,29 @@ * ``GET /workflow/runs/{run_id}`` — fetch a run. * ``POST /workflow/runs/{run_id}/transition``— apply an FSM event and return the run. * ``GET /workflow/runs/{run_id}/red-team`` — Red-Team Gate verdict + evidence. +* ``POST /workflow/runs/{run_id}/red-team`` — trigger a Red-Team scan for a run. Handlers delegate to a process-wide :class:`~forge_workflow.WorkflowEngineImpl` backed by an in-memory run store (the SQLAlchemy-backed store is swapped in behind the same dependency via ``app.dependency_overrides`` / config). Domain errors map to HTTP: unknown run -> 404; invalid/ambiguous transition -> 409. -The ``red-team`` route reads directly off the append-only ``red_team_record`` +The ``red-team`` GET reads directly off the append-only ``red_team_record`` table (see ``forge_db.redteam``) rather than through the FSM engine: it is workspace-scoped on the row itself (``run_id`` is the same ``WorkflowParams.workflow_run_id`` the Temporal ``FeatureWorkflow`` scans before the human spec gate — see ``forge_workflow.temporal.workflows``), so it needs no engine/ownership lookup and degrades safely (empty history, no leak) for an unscanned or foreign run. + +Red-Team Gate, V1 parity (Task 20): the V1 FSM is a plain transition graph with +no gate hooks, and this router is its only production driver — so when a V1 +transition lands a run in ``spec_review`` (the human spec gate, exactly where +the Temporal spine scans), the handler mints the run's verdict once via the +shared :func:`forge_workflow.red_team_gate.ensure_red_team_verdict`: the +configured adversary when one is wired (:func:`get_red_team_fn`), an explicit +parked-pass otherwise — park-don't-fake, never silent. The Temporal spine keeps +owning its own scan inside ``FeatureWorkflow.run`` (the mint is V1-engine-only). """ from __future__ import annotations @@ -37,8 +47,8 @@ from forge_api.db import get_db from forge_api.deps import Principal, get_current_principal from forge_api.routers._rbac import require_permission -from forge_api.schemas.red_team import RedTeamGateOut, RedTeamRecordOut -from forge_contracts import WorkflowRun +from forge_api.schemas.red_team import RedTeamGateOut, RedTeamRecordOut, RedTeamTriggerOut +from forge_contracts import WorkflowRun, WorkflowState from forge_db.redteam import RedTeamRepository from forge_workflow import ( AmbiguousTransitionError, @@ -49,6 +59,11 @@ WorkflowEngineImpl, WorkflowRunNotFoundError, ) +from forge_workflow.red_team_gate import ( + RedTeamFn, + ensure_red_team_verdict, + run_and_record_red_team, +) router = APIRouter( prefix="/workflow", @@ -132,8 +147,30 @@ def get_workflow_ownership() -> WorkflowOwnership: return _workflow_ownership_singleton() +def get_red_team_fn() -> RedTeamFn | None: + """The configured Red-Team adversary harness (override in tests / deploys via DI). + + ``None`` (the default — no adversary model/sandbox is wired in the API + process) makes every V1/triggered scan an EXPLICIT parked-pass + (``kind="parked"``, evidence naming the reason) — park-don't-fake, mirroring + the Temporal activity's ``_default_red_team``. A real deployment overrides + this with a callable that runs the heterogeneous, sandboxed adversary + (``forge_coordinator.red_team.run_red_team``). + """ + return None + + EngineDep = Annotated[WorkflowEngineImpl, Depends(get_workflow_engine)] OwnershipDep = Annotated[WorkflowOwnership, Depends(get_workflow_ownership)] +RedTeamFnDep = Annotated[RedTeamFn | None, Depends(get_red_team_fn)] + +#: Run states at which a red-team scan may be triggered, mapped to the gate +#: phase the scan runs before (mirrors ``RedTeamInput.phase``: spec | pr). +_GATEABLE_STATES: dict[str, str] = { + WorkflowState.SPEC_REVIEW.value: "spec", + WorkflowState.PR_OPENED.value: "pr", + WorkflowState.AWAITING_REVIEW.value: "pr", +} # --------------------------------------------------------------------------- # @@ -185,6 +222,36 @@ def _require_owned( ) +def _run_task_id(run: WorkflowRun) -> uuid.UUID: + """Narrow the contract's optional ``task_id`` (always set by ``start_run``). + + A run without one cannot be red-team scanned (``RedTeamInput.task_id`` is + required) — fail loud rather than silently skipping the gate. + """ + if run.task_id is None: # pragma: no cover — start_run always sets it + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"workflow run {run.id} has no task_id; cannot red-team scan", + ) + return run.task_id + + +def _run_id(run: WorkflowRun) -> uuid.UUID: + """Narrow the contract's optional ``id`` (always set by ``start_run``). + + Mirrors :func:`_run_task_id`: a run reaching the mint gate without an id + cannot be scanned or recorded (``ensure_red_team_verdict`` requires + ``workflow_run_id``) — fail loud (500) rather than silently skipping the + gate. Previously this case was a quiet ``if run.id is not None:`` skip. + """ + if run.id is None: # pragma: no cover — start_run always sets it + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="workflow run has no id; cannot red-team scan", + ) + return run.id + + @router.post("/runs", response_model=WorkflowRun, status_code=status.HTTP_201_CREATED) def start_run( engine: EngineDep, @@ -213,15 +280,45 @@ def get_run( def transition( engine: EngineDep, ownership: OwnershipDep, + session: SessionDep, principal: WriterDep, + red_team_fn: RedTeamFnDep, run_id: uuid.UUID, request: TransitionRequest, ) -> WorkflowRun: - """Apply an FSM transition event and return the updated run.""" + """Apply an FSM transition event and return the updated run. + + Red-Team Gate (V1 parity): when a V1 (FSM-engine) transition lands the run + in ``spec_review`` — the human spec gate, exactly where the Temporal spine + scans — mint the run's verdict once (idempotent across gate re-entries): + the configured adversary when wired, an explicit parked-pass otherwise. + Failure to record is loud (500 via ``_run_id``/``_run_task_id``), never a + silently skipped gate. The Temporal engine is excluded: its workflow body + owns the scan. Recovery from a mint failure needs no special-casing: the + FSM transition itself has already landed (it is a separate store from this + mint), so the next transition that re-enters ``spec_review`` re-attempts + the mint (``ensure_red_team_verdict`` is idempotent on "no existing + record"); alternatively, ``POST /workflow/runs/{run_id}/red-team`` records + the verdict directly as a manual recovery path. + """ _require_owned(ownership, run_id, principal.workspace_id) with _workflow_errors(): engine.transition(run_id, request.event) - return engine.get_run(run_id) + run = engine.get_run(run_id) + if ( + isinstance(engine, WorkflowEngineImpl) + and run.current_state == WorkflowState.SPEC_REVIEW.value + ): + ensure_red_team_verdict( + session, + principal.workspace_id, + workflow_run_id=_run_id(run), + task_id=_run_task_id(run), + phase=_GATEABLE_STATES[run.current_state], + red_team_fn=red_team_fn, + ) + session.commit() + return run @router.get("/runs/{run_id}/red-team", response_model=RedTeamGateOut) @@ -245,10 +342,66 @@ def get_run_red_team( ) +@router.post( + "/runs/{run_id}/red-team", + response_model=RedTeamTriggerOut, + status_code=status.HTTP_202_ACCEPTED, +) +def trigger_run_red_team( + engine: EngineDep, + ownership: OwnershipDep, + session: SessionDep, + principal: WriterDep, + red_team_fn: RedTeamFnDep, + run_id: uuid.UUID, +) -> RedTeamTriggerOut: + """Trigger a Red-Team scan for a run and append its verdict to the history. + + Runs the configured adversary when one is wired (:func:`get_red_team_fn`); + otherwise records an EXPLICIT parked-pass (``kind="parked"``, evidence + naming the missing adversary) — never disguised as a real adversarial pass. + Each trigger appends a fresh ``red_team_record`` (a ``blocked`` scan + followed by a re-triggered ``survived`` one is the documented history the + GET surface returns). ``409`` when the run is not at a gateable state + (``spec_review`` for the spec phase; ``pr_opened``/``awaiting_review`` for + the pr phase); unknown or foreign runs read as ``404`` (no existence leak). + """ + _require_owned(ownership, run_id, principal.workspace_id) + with _workflow_errors(): + run = engine.get_run(run_id) + phase = _GATEABLE_STATES.get(run.current_state) + if phase is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + f"run {run_id} is not at a red-team gateable state " + f"(current_state={run.current_state!r}; gateable: " + f"{', '.join(sorted(_GATEABLE_STATES))})" + ), + ) + row = run_and_record_red_team( + session, + principal.workspace_id, + workflow_run_id=run_id, + task_id=_run_task_id(run), + phase=phase, + red_team_fn=red_team_fn, + actor_id=principal.user_id, + ) + session.commit() + return RedTeamTriggerOut( + workflow_run_id=run_id, + record_id=row.id, + verdict=row.verdict, + kind=row.kind, + ) + + __all__ = [ "StartRunRequest", "TransitionRequest", "WorkflowOwnership", + "get_red_team_fn", "get_workflow_engine", "get_workflow_ownership", "router", diff --git a/apps/api/forge_api/schemas/ao_settings.py b/apps/api/forge_api/schemas/ao_settings.py index 895cddaf..6b16732c 100644 --- a/apps/api/forge_api/schemas/ao_settings.py +++ b/apps/api/forge_api/schemas/ao_settings.py @@ -1,11 +1,13 @@ """Request/response schemas for the Adaptive Orchestration settings API (``ao-settings-api``): per-role model+effort config, the workspace-wide -``tier -> model`` map / complexity thresholds / auto-route toggle, and a -routing-preview endpoint. +``tier -> model`` map / complexity thresholds / auto-route toggle, a +routing-preview endpoint, and the Self-Eval Gate status/run surface. """ from __future__ import annotations +from datetime import datetime +from typing import Literal from uuid import UUID from pydantic import BaseModel, Field @@ -22,6 +24,10 @@ "RoleConfigUpsertRequest", "RoutingPreviewRequest", "RoutingPreviewResponse", + "SelfEvalBaselineOut", + "SelfEvalRunAccepted", + "SelfEvalStatusOut", + "SelfEvalSuiteOut", ] @@ -110,3 +116,48 @@ class RoutingPreviewResponse(BaseModel): junior_max: int medior_max: int auto_route_enabled: bool + + +class SelfEvalSuiteOut(BaseModel): + """The workspace's private Self-Eval suite (case content is never exposed).""" + + id: UUID + slug: str + version: str + title: str + task_count: int + repo_id: str | None + published: bool + + +class SelfEvalBaselineOut(BaseModel): + """The frozen baseline the Self-Eval Gate blocks regressions against.""" + + benchmark_suite_id: UUID + baseline_rate: float + resolved: int + total: int + #: When the baseline row was last minted/refreshed by a scoring run. + recorded_at: datetime + + +class SelfEvalStatusOut(BaseModel): + """Body for ``GET /ao/self-eval/status`` — raw facts, no derived verdicts. + + ``suite``/``baseline`` are ``None`` on cold start; ``enforced`` mirrors the + ``self_eval_enforce`` app setting. The UI derives gate status from these. + """ + + workspace_id: UUID + enforced: bool + suite: SelfEvalSuiteOut | None + baseline: SelfEvalBaselineOut | None + + +class SelfEvalRunAccepted(BaseModel): + """Body for ``POST /ao/self-eval/runs`` (202): the run is queued, not done.""" + + status: Literal["queued"] = "queued" + task: str + workspace_id: UUID + benchmark_suite_id: UUID diff --git a/apps/api/forge_api/schemas/attestation.py b/apps/api/forge_api/schemas/attestation.py new file mode 100644 index 00000000..8d6f477f --- /dev/null +++ b/apps/api/forge_api/schemas/attestation.py @@ -0,0 +1,74 @@ +"""Response schemas for the Attested Changesets read-only REST surface +(``GET /attestations``, ``GET /attestations/{id}``, +``GET /approvals/{id}/attestation`` — Task 19). + +Field mapping is truthful to ``forge_db.models.attestation.Attestation``: +``changeset_hash`` is the row's ``subject_digest`` (the labeled sha256 of the +attested subject), ``keyid``/``predicate_type``/``payload_hash``/``created_at`` +are the columns verbatim, and ``provenance`` carries the queryable provenance +columns the dashboard filters on. ``verified`` is **computed, never stored**: +the router runs the exact ``forge-verify --run`` verification seam +(:func:`forge_api.cli_verify.verify_stored_attestation` — PAE re-derivation + +Ed25519 over the envelope) against the deployment's verification key, so a +record this deployment cannot vouch for honestly reads ``verified: false``. + +The raw DSSE ``envelope`` is deliberately not exposed here: independent +(offline) verification goes through ``forge-verify``, which re-reads the row — +the REST surface never becomes a second source of truth for the signature. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +__all__ = ["AttestationListResponse", "AttestationOut", "AttestationProvenance"] + + +class AttestationProvenance(BaseModel): + """The queryable provenance columns of one ``attestation`` row.""" + + model_config = ConfigDict(from_attributes=True) + + workflow_run_id: uuid.UUID | None = None + agent_run_id: uuid.UUID | None = None + #: PR numbers this attestation covers (mirrors + #: ``TraceabilityCriterionLink.pr_numbers``). + pr_numbers: list[int] = Field(default_factory=list) + #: Spec identity the changeset was produced against; the service degrades + #: honestly to ``""`` / ``0`` when no traceability exists. + spec_key: str | None = None + spec_version: int | None = None + #: Position of the chained ``changeset.attested`` F39 audit event. + audit_seq: int | None = None + + +class AttestationOut(BaseModel): + """One immutable DSSE-signed changeset attestation, with live verification.""" + + id: uuid.UUID + #: The attested subject's labeled digest (``Attestation.subject_digest``, + #: ``sha256:``) — the changeset identity the envelope signs over. + changeset_hash: str + #: The in-toto predicate type URI of the signed Statement. + predicate_type: str + #: sha256 of the raw Ed25519 public key that signed the envelope. + keyid: str + #: sha256 hex of the canonical (PAE-encoded) payload the signature covers. + payload_hash: str + created_at: datetime + #: Computed by the same seam ``forge-verify --run`` uses: recorded + #: ``payload_hash`` matches the PAE re-derivation AND the Ed25519 signature + #: verifies against this deployment's verification key. + verified: bool + provenance: AttestationProvenance + + +class AttestationListResponse(BaseModel): + """Body of ``GET /attestations`` — one workspace-scoped page, newest first.""" + + items: list[AttestationOut] = Field(default_factory=list) + limit: int + offset: int diff --git a/apps/api/forge_api/schemas/red_team.py b/apps/api/forge_api/schemas/red_team.py index 3a41d4f2..5cd2a544 100644 --- a/apps/api/forge_api/schemas/red_team.py +++ b/apps/api/forge_api/schemas/red_team.py @@ -1,5 +1,6 @@ -"""Response schema for the Red-Team Gate surface -(``GET /workflow/runs/{run_id}/red-team``, Red-Team Gate, slice redteam-surface). +"""Response schemas for the Red-Team Gate surface +(``GET``/``POST /workflow/runs/{run_id}/red-team``, Red-Team Gate, +slices redteam-surface + redteam-trigger). """ from __future__ import annotations @@ -10,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field -__all__ = ["RedTeamGateOut", "RedTeamRecordOut"] +__all__ = ["RedTeamGateOut", "RedTeamRecordOut", "RedTeamTriggerOut"] class RedTeamRecordOut(BaseModel): @@ -46,3 +47,16 @@ class RedTeamGateOut(BaseModel): workflow_run_id: uuid.UUID latest: RedTeamRecordOut | None = None records: list[RedTeamRecordOut] = Field(default_factory=list) + + +class RedTeamTriggerOut(BaseModel): + """Acknowledgement of ``POST /workflow/runs/{run_id}/red-team``: the scan ran + and its verdict row was appended. ``record_id`` is the freshly-recorded + ``red_team_record`` the follow-up GET returns as ``latest``. ``kind="parked"`` + means no adversary is configured — an explicit park, never a claimed + adversarial pass.""" + + workflow_run_id: uuid.UUID + record_id: uuid.UUID + verdict: str + kind: str diff --git a/apps/api/forge_api/services/pm_service.py b/apps/api/forge_api/services/pm_service.py index cd6c0b0a..711ae6e1 100644 --- a/apps/api/forge_api/services/pm_service.py +++ b/apps/api/forge_api/services/pm_service.py @@ -6,23 +6,32 @@ dedupes inbound webhooks, and writes an immutable audit entry per accepted webhook / health probe. -Board-write execution (the worker's ``process_webhook`` -> re-fetch -> -``sync_in`` and the outbound ``activity_events`` scan) is intentionally **not** -performed here — see module notes / the slice report: it depends on the F01 -Postgres board substrate (``activity_events`` outbox + versioned task service) -which is not present in this foundation. The engine that performs it lives in -``forge_integrations.pm.sync_engine`` and is fully unit-tested against fakes. +Board-write execution now runs: each accepted (non-skipped) delivery enqueues +the worker task ``forge.pm.process_webhook`` +(``forge_worker.tasks.pm_sync``), which re-fetches the changed issue through +the provider adapter and upserts the board via the F01 Postgres board +substrate (``SqlAlchemyBoardService`` + ``DbLinkRepository``), workspace- +scoped and idempotent on redelivery. Enqueueing is fail-open: a broker outage +logs a loud warning and the webhook still returns 202 (the persisted delivery +row is the durable record for reprocessing). + +Still parked (unchanged scope): OAuth code exchange, historical backfill, +manual conflict-resolve execution, external delete propagation, and the +outbound ``activity_events`` scan. """ from __future__ import annotations import builtins import hashlib +import logging +import os import secrets import time import uuid from collections.abc import Callable from datetime import UTC, datetime +from functools import lru_cache from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker @@ -59,6 +68,29 @@ PMTransport, ) +logger = logging.getLogger(__name__) + +#: Must match ``apps/worker/forge_worker/tasks/pm_sync.py::PM_SYNC_TASK`` — +#: duplicated (not imported) because ``forge-api`` cannot depend on +#: ``forge-worker`` (the dependency runs the other way); mirrors the +#: ``FULL_SYNC_TASK`` pattern in ``mcp_index_service.py``. +PROCESS_WEBHOOK_TASK = "forge.pm.process_webhook" + + +@lru_cache(maxsize=1) +def _celery_app() -> object: # pragma: no cover - prod seam + from celery import Celery + + url = os.environ.get("FORGE_REDIS_URL", "redis://localhost:6379/0") + return Celery("forge-api-enqueue", broker=url, backend=url) + + +def enqueue_process_webhook(delivery_row_id: uuid.UUID) -> None: # pragma: no cover - prod seam + """Enqueue the worker board-write task for one persisted delivery.""" + _celery_app().send_task( # type: ignore[attr-defined] + PROCESS_WEBHOOK_TASK, args=[str(delivery_row_id)] + ) + class PMConnectionNotFound(LookupError): """Raised when a connection id is absent in the caller's workspace.""" @@ -91,11 +123,13 @@ def __init__( vault: SecretVault, audit: AuditLog, transport_factory: Callable[[PMConnection], PMTransport] = _default_transport_factory, + process_webhook_enqueue: Callable[[uuid.UUID], None] = enqueue_process_webhook, ) -> None: self._sf = session_factory self._vault = vault self._audit = audit self._transport_factory = transport_factory + self._enqueue_process_webhook = process_webhook_enqueue # ------------------------------------------------------------------ # # connection CRUD # @@ -322,10 +356,12 @@ def get_connection_any_workspace(self, connection_id: uuid.UUID) -> PMConnection def receive_webhook( self, connection: PMConnection, body: bytes, headers: dict[str, str] ) -> tuple[int, WebhookEvent | None]: - """Verify -> dedupe -> persist a delivery. Returns ``(status_code, event)``. + """Verify -> dedupe -> persist -> enqueue. Returns ``(status_code, event)``. - The payload is a *hint*; the worker re-fetches authoritative state before - any board write (parked — see module docstring). 401 on bad signature. + The payload is a *hint*; the worker task (``forge.pm.process_webhook``) + re-fetches authoritative state before the board write. 401 on bad + signature. Enqueue failures never fail the response (fail-open + warn); + skipped deliveries (disabled / outbound-only) are never enqueued. """ adapter = self._build_adapter(connection) secret = self._webhook_secret(connection) @@ -359,6 +395,7 @@ def receive_webhook( ) session.add(delivery) session.commit() + delivery_row_id = delivery.id self._audit.record( category=AuditCategory.SYSTEM, @@ -368,7 +405,18 @@ def receive_webhook( status=status.value, payload_hash=payload_hash, ) - # NOTE: enqueue of pm.process_webhook is parked (worker board-write path). + if status == PMDeliveryStatus.RECEIVED: + try: + self._enqueue_process_webhook(delivery_row_id) + except Exception: + logger.warning( + "pm webhook accepted but worker enqueue failed " + "(delivery row %s, connection %s); the board write will not " + "run until the delivery is reprocessed", + delivery_row_id, + connection.id, + exc_info=True, + ) return 202, event # ------------------------------------------------------------------ # @@ -474,8 +522,10 @@ def _default_priority_map(provider: PMProvider) -> dict: __all__ = [ + "PROCESS_WEBHOOK_TASK", "PMConflictExists", "PMConnectionNotFound", "PMConnectionService", "PMError", + "enqueue_process_webhook", ] diff --git a/apps/api/forge_api/services/self_eval_gate.py b/apps/api/forge_api/services/self_eval_gate.py index 975c339f..e5c56e86 100644 --- a/apps/api/forge_api/services/self_eval_gate.py +++ b/apps/api/forge_api/services/self_eval_gate.py @@ -13,7 +13,8 @@ API gate has no fresh scorecard for the *proposed* config and no-ops, while the gate MECHANISM (baseline lookup, regression block, force override, audit) is fully wired and exercised in tests by injecting a runner. Establishing/refreshing -a baseline is the worker-owned ``POST /ao/self-eval/runs`` path (A4). +a baseline is the worker-owned ``forge.self_eval.run`` Celery task (A4), which +``POST /ao/self-eval/runs`` enqueues. """ from __future__ import annotations diff --git a/apps/api/forge_api/services/self_eval_service.py b/apps/api/forge_api/services/self_eval_service.py index b8c8e69a..61044ec4 100644 --- a/apps/api/forge_api/services/self_eval_service.py +++ b/apps/api/forge_api/services/self_eval_service.py @@ -11,9 +11,11 @@ from __future__ import annotations +import os import uuid from collections.abc import Mapping from dataclasses import dataclass +from functools import lru_cache from typing import Any from sqlalchemy import select @@ -21,6 +23,42 @@ from forge_db.models.benchmark import SelfEvalBaseline +#: Must match ``apps/worker/forge_worker/tasks/self_eval_run.py``'s task name — +#: duplicated (not imported) because ``forge-api`` cannot depend on +#: ``forge-worker`` (the dependency runs the other way); mirrors the +#: ``PROCESS_WEBHOOK_TASK`` pattern in ``pm_service.py``. +SELF_EVAL_RUN_TASK = "forge.self_eval.run" + + +@lru_cache(maxsize=1) +def _celery_app() -> object: # pragma: no cover - prod seam + from celery import Celery + + url = os.environ.get("FORGE_REDIS_URL", "redis://localhost:6379/0") + return Celery("forge-api-enqueue", broker=url, backend=url) + + +def enqueue_self_eval_run( # pragma: no cover - prod seam + workspace_id: uuid.UUID, + proposed_config: Mapping[str, Any], + *, + recorded_by: uuid.UUID | None = None, +) -> None: + """Enqueue the worker-owned private-suite run (``forge.self_eval.run``). + + The run itself stays in the worker (minutes-long, agent-driven); this is the + thin API->worker seam the settings panel's "run self-eval" action goes + through. ``proposed_config`` must already be redacted (no secrets). + """ + _celery_app().send_task( # type: ignore[attr-defined] + SELF_EVAL_RUN_TASK, + args=[ + str(workspace_id), + dict(proposed_config), + str(recorded_by) if recorded_by else None, + ], + ) + @dataclass(frozen=True) class BaselineRecord: diff --git a/apps/api/tests/attest/test_attestation_api.py b/apps/api/tests/attest/test_attestation_api.py new file mode 100644 index 00000000..3d440dc2 --- /dev/null +++ b/apps/api/tests/attest/test_attestation_api.py @@ -0,0 +1,376 @@ +"""Integration tests for the Attested Changesets read-only REST surface +(Task 19: ``GET /attestations``, ``GET /attestations/{id}``, +``GET /approvals/{id}/attestation``). + +Mints real records through :class:`AttestationService` (the same path the +``pr``-gate approval hook uses — mirroring ``test_attestation_service.py``'s +seeding) on hermetic SQLite (mirrors ``test_red_team_api.py``'s convention: the +Postgres immutability trigger is a no-op here; the append-only property itself +is proven by the service tests). ``verified`` must be computed by the exact +verification path ``forge-verify --run`` uses — re-derive ``payload_hash`` from +the envelope's PAE, then Ed25519-verify against the deployment key — so a +record signed by a *different* key honestly reads ``verified: false``. +""" + +from __future__ import annotations + +import base64 +import uuid +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import StaticPool, create_engine +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.db import get_db +from forge_api.deps import Principal, get_current_principal +from forge_api.main import create_app +from forge_api.services.approval_service import get_approval_service +from forge_api.services.attestation_service import AttestationService +from forge_approval import ( + ApprovalAuthorizer, + ApprovalService, + GateRegistry, + InMemoryApprovalRepository, +) +from forge_contracts import UserRole +from forge_db.base import Base +from forge_db.models import AgentRun, Attestation, Project, Task, WorkflowRun, Workspace +from forge_obs.attest.signing import DsseSigner, EnvSigningKeyProvider + +WS = uuid.UUID("00000000-0000-0000-0000-0000000000e1") +WS2 = uuid.UUID("00000000-0000-0000-0000-0000000000e2") + +#: Fixed 32-byte Ed25519 seeds → deterministic, silent signers. ``_SEED_B64`` is +#: also exported as ``FORGE_ATTEST_SIGNING_KEY`` (autouse below), so the REST +#: surface's env-fallback verification key matches records minted with it — +#: and does NOT match records minted with ``_OTHER_SEED_B64``. +_SEED_B64 = base64.b64encode(bytes(range(1, 33))).decode("ascii") +_OTHER_SEED_B64 = base64.b64encode(bytes(range(33, 65))).decode("ascii") + + +@pytest.fixture(autouse=True) +def _deployment_signing_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FORGE_ATTEST_SIGNING_KEY", _SEED_B64) + + +@pytest.fixture +def signer() -> DsseSigner: + return DsseSigner(EnvSigningKeyProvider(environ={"FORGE_ATTEST_SIGNING_KEY": _SEED_B64})) + + +@pytest.fixture +def other_signer() -> DsseSigner: + return DsseSigner(EnvSigningKeyProvider(environ={"FORGE_ATTEST_SIGNING_KEY": _OTHER_SEED_B64})) + + +@pytest.fixture +def factory() -> Iterator[sessionmaker[Session]]: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + Base.metadata.create_all(engine) + sf: sessionmaker[Session] = sessionmaker(bind=engine, expire_on_commit=False, class_=Session) + with sf() as s: + s.add(Workspace(id=WS, name="Acme", slug="acme")) + s.add(Workspace(id=WS2, name="Rival", slug="rival")) + s.commit() + yield sf + engine.dispose() + + +def _seed_run(factory: sessionmaker[Session], *, workspace_id: uuid.UUID = WS) -> uuid.UUID: + """Seed workspace -> project -> task -> workflow_run + agent_run; return run id.""" + with factory() as s: + project = Project(workspace_id=workspace_id, name="Core", key=f"C{uuid.uuid4().hex[:4]}") + s.add(project) + s.flush() + task = Task( + workspace_id=workspace_id, + project_id=project.id, + key=f"TASK-{uuid.uuid4().hex[:6]}", + title="attested changeset task", + ) + s.add(task) + s.flush() + run = WorkflowRun(workspace_id=workspace_id, task_id=task.id) + s.add(run) + s.flush() + s.add( + AgentRun( + workspace_id=workspace_id, + workflow_run_id=run.id, + task_id=task.id, + role="implementer", + model="claude-sonnet-4-5", + sandbox_kind="gvisor", + steps=[{"index": 0, "kind": "tool_call", "tool_call": {"tool": "edit_file"}}], + output={"artifacts": {"model_usage": {"version": "2025-09-29"}}}, + ) + ) + s.commit() + run_id = run.id + return run_id + + +def _mint( + factory: sessionmaker[Session], + signer: DsseSigner, + run_id: uuid.UUID, + *, + pr_numbers: list[int] | None = None, + created_at: datetime | None = None, +) -> uuid.UUID: + """Mint one attestation via the real service (the approval hook's path).""" + with factory() as s: + row = AttestationService(s, signer=signer).attest_changeset( + run_id, pr_numbers=pr_numbers if pr_numbers is not None else [7, 9] + ) + if created_at is not None: + # Deterministic ordering on SQLite (second-resolution timestamps); + # the Postgres trigger forbidding this is proven by the service tests. + s.query(Attestation).filter(Attestation.id == row.id).update({"created_at": created_at}) + s.commit() + att_id = row.id + return att_id + + +def _principal(workspace_id: uuid.UUID = WS) -> Principal: + return Principal( + user_id=uuid.uuid4(), + workspace_id=workspace_id, + role=UserRole.MEMBER, + email="member@acme.test", + auth_method="test", + scopes=["*"], + ) + + +def _client( + factory: sessionmaker[Session], + principal: Principal, + approval_service: ApprovalService | None = None, +) -> TestClient: + app: FastAPI = create_app() + + def _get_db() -> Iterator[Session]: + with factory() as session: + yield session + + app.dependency_overrides[get_db] = _get_db + app.dependency_overrides[get_current_principal] = lambda: principal + if approval_service is not None: + app.dependency_overrides[get_approval_service] = lambda: approval_service + return TestClient(app) + + +def _approval_service() -> ApprovalService: + """A hermetic approval service (no resolution hooks — read/create only).""" + return ApprovalService(InMemoryApprovalRepository(), GateRegistry(), ApprovalAuthorizer()) + + +# --------------------------------------------------------------------------- # +# GET /attestations (list) # +# --------------------------------------------------------------------------- # + + +def test_list_returns_minted_record_with_verified_true(factory, signer) -> None: + run_id = _seed_run(factory) + att_id = _mint(factory, signer, run_id) + client = _client(factory, _principal()) + + resp = client.get("/attestations") + assert resp.status_code == 200, resp.text + items = resp.json()["items"] + assert len(items) == 1 + item = items[0] + + assert item["id"] == str(att_id) + assert item["changeset_hash"].startswith("sha256:") + assert item["keyid"] == signer.keyid + assert item["verified"] is True + assert item["created_at"] is not None + assert item["predicate_type"].startswith("https://") + # sha256 hex of the PAE-encoded payload (64 hex chars). + assert len(item["payload_hash"]) == 64 + # Provenance mirrors the queryable model columns, truthfully degraded + # (no traceability seeded -> spec_key "" / spec_version 0). + prov = item["provenance"] + assert prov["workflow_run_id"] == str(run_id) + assert prov["agent_run_id"] is not None + assert prov["pr_numbers"] == [7, 9] + assert prov["spec_key"] == "" + assert prov["spec_version"] == 0 + assert prov["audit_seq"] is not None + + +def test_list_is_newest_first_and_paginates(factory, signer) -> None: + now = datetime.now(UTC) + run_ids = [_seed_run(factory) for _ in range(3)] + att_ids = [ + _mint(factory, signer, run_id, created_at=now + timedelta(minutes=i)) + for i, run_id in enumerate(run_ids) + ] + client = _client(factory, _principal()) + + resp = client.get("/attestations", params={"limit": 2}) + assert resp.status_code == 200, resp.text + page_one = resp.json()["items"] + assert [i["id"] for i in page_one] == [str(att_ids[2]), str(att_ids[1])] + + resp = client.get("/attestations", params={"limit": 2, "offset": 2}) + assert resp.status_code == 200, resp.text + page_two = resp.json()["items"] + assert [i["id"] for i in page_two] == [str(att_ids[0])] + + +def test_list_filters_by_workflow_run_id(factory, signer) -> None: + run_a = _seed_run(factory) + run_b = _seed_run(factory) + att_a = _mint(factory, signer, run_a) + _mint(factory, signer, run_b) + client = _client(factory, _principal()) + + resp = client.get("/attestations", params={"workflow_run_id": str(run_a)}) + assert resp.status_code == 200, resp.text + items = resp.json()["items"] + assert [i["id"] for i in items] == [str(att_a)] + + +def test_list_never_leaks_cross_workspace_records(factory, signer) -> None: + """The row-level workspace scope, not record existence, is the boundary.""" + own_run = _seed_run(factory, workspace_id=WS) + foreign_run = _seed_run(factory, workspace_id=WS2) + own_att = _mint(factory, signer, own_run) + foreign_att = _mint(factory, signer, foreign_run) + + resp = _client(factory, _principal(workspace_id=WS)).get("/attestations") + assert resp.status_code == 200, resp.text + assert [i["id"] for i in resp.json()["items"]] == [str(own_att)] + + resp = _client(factory, _principal(workspace_id=WS2)).get("/attestations") + assert resp.status_code == 200, resp.text + assert [i["id"] for i in resp.json()["items"]] == [str(foreign_att)] + + +def test_list_verification_failure_is_reported_honestly(factory, other_signer) -> None: + """A record signed by a key that is NOT the deployment's verification key + reads ``verified: false`` — never a fake pass.""" + run_id = _seed_run(factory) + _mint(factory, other_signer, run_id) + client = _client(factory, _principal()) + + resp = client.get("/attestations") + assert resp.status_code == 200, resp.text + item = resp.json()["items"][0] + assert item["keyid"] == other_signer.keyid + assert item["verified"] is False + + +# --------------------------------------------------------------------------- # +# GET /attestations/{id} (detail) # +# --------------------------------------------------------------------------- # + + +def test_detail_returns_one_record(factory, signer) -> None: + run_id = _seed_run(factory) + att_id = _mint(factory, signer, run_id) + client = _client(factory, _principal()) + + resp = client.get(f"/attestations/{att_id}") + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["id"] == str(att_id) + assert body["verified"] is True + assert body["provenance"]["workflow_run_id"] == str(run_id) + + +def test_detail_unknown_id_404s(factory) -> None: + client = _client(factory, _principal()) + resp = client.get(f"/attestations/{uuid.uuid4()}") + assert resp.status_code == 404 + + +def test_detail_foreign_workspace_id_404s(factory, signer) -> None: + """A cross-workspace id looks nonexistent (no existence leak).""" + run_id = _seed_run(factory, workspace_id=WS) + att_id = _mint(factory, signer, run_id) + + resp = _client(factory, _principal(workspace_id=WS2)).get(f"/attestations/{att_id}") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- # +# GET /approvals/{id}/attestation (by-approval lookup) # +# --------------------------------------------------------------------------- # + + +def _open_pr_gate( + client: TestClient, *, workflow_run_id: uuid.UUID | None, subject_id: uuid.UUID | None = None +) -> str: + resp = client.post( + "/approvals", + json={ + "gate_type": "pr", + "subject_type": "workflow_run", + "subject_id": str(subject_id or workflow_run_id or uuid.uuid4()), + "workflow_run_id": str(workflow_run_id) if workflow_run_id else None, + "title": "PR gate", + }, + ) + assert resp.status_code == 201, resp.text + return resp.json()["id"] + + +def test_by_approval_returns_the_runs_attestation(factory, signer) -> None: + run_id = _seed_run(factory) + att_id = _mint(factory, signer, run_id) + client = _client(factory, _principal(), approval_service=_approval_service()) + approval_id = _open_pr_gate(client, workflow_run_id=run_id) + + resp = client.get(f"/approvals/{approval_id}/attestation") + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["id"] == str(att_id) + assert body["verified"] is True + assert body["provenance"]["workflow_run_id"] == str(run_id) + + +def test_by_approval_404_when_gate_has_no_workflow_run(factory) -> None: + client = _client(factory, _principal(), approval_service=_approval_service()) + approval_id = _open_pr_gate(client, workflow_run_id=None) + + resp = client.get(f"/approvals/{approval_id}/attestation") + assert resp.status_code == 404 + + +def test_by_approval_404_when_run_is_unattested(factory) -> None: + run_id = _seed_run(factory) + client = _client(factory, _principal(), approval_service=_approval_service()) + approval_id = _open_pr_gate(client, workflow_run_id=run_id) + + resp = client.get(f"/approvals/{approval_id}/attestation") + assert resp.status_code == 404 + + +def test_by_approval_unknown_approval_404s(factory) -> None: + client = _client(factory, _principal(), approval_service=_approval_service()) + resp = client.get(f"/approvals/{uuid.uuid4()}/attestation") + assert resp.status_code == 404 + + +def test_by_approval_cross_workspace_404s(factory, signer) -> None: + """A foreign workspace's approval id looks nonexistent — same contract as + the approvals router itself.""" + run_id = _seed_run(factory, workspace_id=WS) + _mint(factory, signer, run_id) + service = _approval_service() + owner_client = _client(factory, _principal(workspace_id=WS), approval_service=service) + approval_id = _open_pr_gate(owner_client, workflow_run_id=run_id) + + foreign_client = _client(factory, _principal(workspace_id=WS2), approval_service=service) + resp = foreign_client.get(f"/approvals/{approval_id}/attestation") + assert resp.status_code == 404 diff --git a/apps/api/tests/pm_api/conftest.py b/apps/api/tests/pm_api/conftest.py index 23b8fa5c..7af395da 100644 --- a/apps/api/tests/pm_api/conftest.py +++ b/apps/api/tests/pm_api/conftest.py @@ -105,15 +105,25 @@ def audit() -> AuditLog: return AuditLog() +@pytest.fixture +def enqueued() -> list[uuid.UUID]: + """Delivery row ids handed to the worker-enqueue seam (recorded, no broker).""" + return [] + + @pytest.fixture def pm_service( - session_factory: sessionmaker[Session], vault: SecretVault, audit: AuditLog + session_factory: sessionmaker[Session], + vault: SecretVault, + audit: AuditLog, + enqueued: list[uuid.UUID], ) -> PMConnectionService: return PMConnectionService( session_factory=session_factory, vault=vault, audit=audit, transport_factory=_transport_factory, + process_webhook_enqueue=enqueued.append, ) diff --git a/apps/api/tests/pm_api/test_webhooks.py b/apps/api/tests/pm_api/test_webhooks.py index cb47ebf4..ce2fafdd 100644 --- a/apps/api/tests/pm_api/test_webhooks.py +++ b/apps/api/tests/pm_api/test_webhooks.py @@ -174,6 +174,78 @@ def test_outbound_only_records_skipped( assert row.status == PMDeliveryStatus.SKIPPED +def _signed_linear(pm_service, vault, conn: dict, *, webhook_id: str) -> tuple[bytes, dict]: + secret = _webhook_secret(pm_service, uuid.UUID(conn["id"]), vault) + ts = int(datetime.now(UTC).timestamp() * 1000) + body = json.dumps( + { + "action": "update", + "type": "Issue", + "webhookTimestamp": ts, + "webhookId": webhook_id, + "data": {"id": "uuid-1", "identifier": "ENG-1"}, + } + ).encode() + sig = sign_linear(secret, body) + return body, {"Linear-Signature": sig, "Content-Type": "application/json"} + + +# --- Worker enqueue (board-write path) -------------------------------------- # + + +def test_webhook_enqueues_board_write_once( + client, project_id, pm_service, vault, session_factory, enqueued +) -> None: + """Post-persist the intake enqueues the worker task with the delivery row id; + a provider redelivery (same delivery id) never enqueues twice.""" + conn = _create(client, project_id, "linear") + body, headers = _signed_linear(pm_service, vault, conn, webhook_id="wh-enq-1") + url = f"/integrations/pm/webhooks/linear/{conn['id']}" + + assert client.post(url, content=body, headers=headers).status_code == 202 + assert client.post(url, content=body, headers=headers).status_code == 202 + + from forge_db.models.pm import PMWebhookDelivery + + with session_factory() as session: + row = session.query(PMWebhookDelivery).one() + assert enqueued == [row.id] + + +def test_webhook_enqueue_failure_still_202( + client, project_id, pm_service, vault, monkeypatch, caplog +) -> None: + """A broken broker must never fail the webhook response (fail-open + warn).""" + import logging + + conn = _create(client, project_id, "linear") + body, headers = _signed_linear(pm_service, vault, conn, webhook_id="wh-enq-2") + + def _boom(_row_id: uuid.UUID) -> None: + raise RuntimeError("broker down") + + monkeypatch.setattr(pm_service, "_enqueue_process_webhook", _boom) + with caplog.at_level(logging.WARNING, logger="forge_api.services.pm_service"): + resp = client.post( + f"/integrations/pm/webhooks/linear/{conn['id']}", content=body, headers=headers + ) + assert resp.status_code == 202 + assert "enqueue failed" in caplog.text + + +def test_webhook_skipped_delivery_not_enqueued( + client, project_id, pm_service, vault, enqueued +) -> None: + """Outbound-only connections persist a skipped delivery and never enqueue.""" + conn = _create(client, project_id, "linear", sync_direction="outbound_only") + body, headers = _signed_linear(pm_service, vault, conn, webhook_id="wh-enq-3") + resp = client.post( + f"/integrations/pm/webhooks/linear/{conn['id']}", content=body, headers=headers + ) + assert resp.status_code == 202 + assert enqueued == [] + + def test_webhook_unknown_connection_404(client) -> None: resp = client.post( f"/integrations/pm/webhooks/linear/{uuid.uuid4()}", diff --git a/apps/api/tests/test_ao_settings_router.py b/apps/api/tests/test_ao_settings_router.py index bd095715..4c14bb53 100644 --- a/apps/api/tests/test_ao_settings_router.py +++ b/apps/api/tests/test_ao_settings_router.py @@ -3,9 +3,10 @@ Real handlers over a real Postgres session (``pg_engine``, shared root fixture): per-role model+effort config (list/upsert/delete, workspace vs project override precedence, RBAC, workspace isolation), the workspace-wide -settings (auto-route, tier-model overrides, complexity thresholds, RBAC), and -the routing-preview endpoint (default sizing, custom thresholds, custom -tier-model overrides, invalid provider). +settings (auto-route, tier-model overrides, complexity thresholds, RBAC), the +routing-preview endpoint (default sizing, custom thresholds, custom +tier-model overrides, invalid provider), and the Self-Eval Gate surface +(status read + the admin run trigger that enqueues ``forge.self_eval.run``). """ from __future__ import annotations @@ -16,14 +17,19 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker from forge_api.db import get_db from forge_api.deps import Principal from forge_api.main import create_app +from forge_api.routers import ao_settings as ao_module +from forge_api.services import self_eval_service as self_eval_service_module +from forge_api.settings import Settings from forge_contracts import UserRole from forge_db.base import Base -from forge_db.models import Project, Workspace +from forge_db.models import AuditLog, Project, User, Workspace +from forge_db.models.benchmark import BenchmarkSuite, SelfEvalBaseline pytestmark = pytest.mark.usefixtures("pg_engine") @@ -334,3 +340,201 @@ def test_routing_preview_rejects_invalid_provider( json={"kind": "doc", "priority": "low", "provider": "not-a-provider"}, ) assert resp.status_code == 422 + + +# --------------------------------------------------------------------------- # +# Self-Eval Gate: status read + run trigger # +# --------------------------------------------------------------------------- # + + +def _workspace_of(client: TestClient) -> uuid.UUID: + return uuid.UUID(client.get("/ao/settings").json()["workspace_id"]) + + +def _seed_private_suite( + factory: sessionmaker[Session], workspace_id: uuid.UUID, *, published: bool = True +) -> uuid.UUID: + with factory() as session: + suite = BenchmarkSuite( + slug=f"self-eval-{uuid.uuid4().hex[:8]}", + version="1.0.0", + title="Private self-eval suite", + task_count=10, + content_hash="deadbeef", + frozen=True, + published=published, + workspace_id=workspace_id, + repo_id="github:acme/app", + private=True, + ) + session.add(suite) + session.flush() + suite_id = suite.id + session.commit() + return suite_id + + +def _seed_baseline( + factory: sessionmaker[Session], workspace_id: uuid.UUID, suite_id: uuid.UUID +) -> None: + with factory() as session: + session.add( + SelfEvalBaseline( + workspace_id=workspace_id, + benchmark_suite_id=suite_id, + baseline_rate=0.8, + resolved=8, + total=10, + config={"scope": "ao.settings"}, + ) + ) + session.commit() + + +def test_self_eval_status_cold_start_is_honest_nulls( + client_for: Callable[[UserRole], TestClient], +) -> None: + client = client_for(UserRole.VIEWER) + resp = client.get("/ao/self-eval/status") + assert resp.status_code == 200 + body = resp.json() + assert body["enforced"] is False + assert body["suite"] is None + assert body["baseline"] is None + + +def test_self_eval_status_reports_suite_and_baseline( + client_for: Callable[[UserRole], TestClient], factory: sessionmaker[Session] +) -> None: + client = client_for(UserRole.VIEWER) + workspace_id = _workspace_of(client) + suite_id = _seed_private_suite(factory, workspace_id) + _seed_baseline(factory, workspace_id, suite_id) + + body = client.get("/ao/self-eval/status").json() + assert body["suite"]["id"] == str(suite_id) + assert body["suite"]["published"] is True + assert body["suite"]["repo_id"] == "github:acme/app" + assert body["baseline"]["benchmark_suite_id"] == str(suite_id) + assert body["baseline"]["baseline_rate"] == 0.8 + assert (body["baseline"]["resolved"], body["baseline"]["total"]) == (8, 10) + assert body["baseline"]["recorded_at"] + + +def test_self_eval_status_reflects_enforcement_flag( + client_for: Callable[[UserRole], TestClient], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(ao_module, "get_app_settings", lambda: Settings(self_eval_enforce=True)) + client = client_for(UserRole.VIEWER) + assert client.get("/ao/self-eval/status").json()["enforced"] is True + + +def test_self_eval_status_is_workspace_isolated( + factory: sessionmaker[Session], authenticate_app: Callable[..., FastAPI] +) -> None: + ws_a = _seed_workspace(factory, name="A", slug=f"a-{uuid.uuid4().hex[:8]}") + ws_b = _seed_workspace(factory, name="B", slug=f"b-{uuid.uuid4().hex[:8]}") + suite_a = _seed_private_suite(factory, ws_a) + _seed_baseline(factory, ws_a, suite_a) + + app = create_app() + authenticate_app(app, _principal(UserRole.VIEWER, ws_b)) + + def _get_db() -> Iterator[Session]: + with factory() as session: + yield session + + app.dependency_overrides[get_db] = _get_db + body = TestClient(app).get("/ao/self-eval/status").json() + assert body["suite"] is None + assert body["baseline"] is None + + +def test_viewer_cannot_request_self_eval_run( + client_for: Callable[[UserRole], TestClient], +) -> None: + client = client_for(UserRole.VIEWER) + assert client.post("/ao/self-eval/runs").status_code == 403 + + +def test_self_eval_run_without_private_suite_is_409( + client_for: Callable[[UserRole], TestClient], +) -> None: + client = client_for(UserRole.ADMIN) + resp = client.post("/ao/self-eval/runs") + assert resp.status_code == 409 + assert resp.json()["detail"]["error"] == "no_private_suite" + + +def test_self_eval_run_unpublished_suite_is_409( + client_for: Callable[[UserRole], TestClient], factory: sessionmaker[Session] +) -> None: + client = client_for(UserRole.ADMIN) + _seed_private_suite(factory, _workspace_of(client), published=False) + resp = client.post("/ao/self-eval/runs") + assert resp.status_code == 409 + assert resp.json()["detail"]["error"] == "no_private_suite" + + +def test_admin_self_eval_run_enqueues_worker_task_and_audits( + factory: sessionmaker[Session], + authenticate_app: Callable[..., FastAPI], + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Self-contained client: the run-request audit row carries a real actor FK, + # so the principal must be a persisted user (unlike the shared client_for). + workspace_id = _seed_workspace(factory, name="Run", slug=f"run-{uuid.uuid4().hex[:8]}") + with factory() as session: + admin = User(workspace_id=workspace_id, email="admin@forge.local", role=UserRole.ADMIN) + session.add(admin) + session.flush() + user_id = admin.id + session.commit() + + app = create_app() + principal = Principal( + user_id=user_id, + workspace_id=workspace_id, + role=UserRole.ADMIN, + email="admin@forge.local", + auth_method="test", + scopes=["*"], + ) + authenticate_app(app, principal) + + def _get_db() -> Iterator[Session]: + with factory() as session: + yield session + + app.dependency_overrides[get_db] = _get_db + client = TestClient(app) + suite_id = _seed_private_suite(factory, workspace_id) + + calls: list[tuple[tuple, dict]] = [] + monkeypatch.setattr( + self_eval_service_module, + "enqueue_self_eval_run", + lambda *args, **kwargs: calls.append((args, kwargs)), + ) + + resp = client.post("/ao/self-eval/runs") + assert resp.status_code == 202 + body = resp.json() + assert body["status"] == "queued" + assert body["task"] == "forge.self_eval.run" + assert body["benchmark_suite_id"] == str(suite_id) + + assert len(calls) == 1 + (ws_arg, config_arg), kwargs = calls[0] + assert ws_arg == workspace_id + assert config_arg["scope"] == "ao.settings" + assert "auto_route" in config_arg + assert kwargs["recorded_by"] is not None + + with factory() as session: + actions = list( + session.scalars( + select(AuditLog.action).where(AuditLog.workspace_id == workspace_id) + ).all() + ) + assert "ao.self_eval.run_requested" in actions diff --git a/apps/api/tests/test_api_skeleton.py b/apps/api/tests/test_api_skeleton.py index c159a157..538d6e7a 100644 --- a/apps/api/tests/test_api_skeleton.py +++ b/apps/api/tests/test_api_skeleton.py @@ -40,7 +40,7 @@ "/policy", "/mcp", "/integration", - "/approval", + "/approvals", "/observability", "/auth", ] diff --git a/apps/api/tests/test_approval_router.py b/apps/api/tests/test_approval_router.py deleted file mode 100644 index 88b88bb5..00000000 --- a/apps/api/tests/test_approval_router.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Integration tests for the approval router (Phase 2 Task 2.1 wires ``/approval/*``). - -Exercises the real handlers wired to a fresh in-memory :class:`ApprovalStore`: -create a request, list/get it, and record a decision; unknown ids -> 404. -""" - -from __future__ import annotations - -import uuid -from collections.abc import Callable, Iterator - -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from forge_api.main import create_app -from forge_api.routers.approval import ApprovalStore, get_approval_store - - -@pytest.fixture -def client(authenticate_app: Callable[..., FastAPI]) -> Iterator[TestClient]: - app = create_app() - authenticate_app(app) - store = ApprovalStore() - app.dependency_overrides[get_approval_store] = lambda: store - with TestClient(app) as c: - yield c - - -def _create(client: TestClient) -> dict: - resp = client.post( - "/approval/requests", - json={"gate": "pr", "title": "Approve PR for TASK-1", "confidence": 0.8}, - ) - assert resp.status_code == 201, resp.text - return resp.json() - - -def test_create_and_get(client: TestClient) -> None: - created = _create(client) - assert created["id"] - assert created["status"] == "pending" - fetched = client.get(f"/approval/requests/{created['id']}") - assert fetched.status_code == 200 - assert fetched.json()["title"] == "Approve PR for TASK-1" - - -def test_list_requests(client: TestClient) -> None: - _create(client) - resp = client.get("/approval/requests") - assert resp.status_code == 200 - assert len(resp.json()) == 1 - pending = client.get("/approval/requests", params={"status": "pending"}) - assert len(pending.json()) == 1 - - -def test_decide_approves(client: TestClient) -> None: - created = _create(client) - resp = client.post( - f"/approval/requests/{created['id']}/decision", - # A forged ``decided_by`` in the body must be ignored: the decider is the - # authenticated principal, not whatever the caller claims. - json={"status": "approved", "decided_by": "alice", "reason": "looks good"}, - ) - assert resp.status_code == 200, resp.text - body = resp.json() - assert body["status"] == "approved" - assert body["decided_by"] == "test-principal@forge.local" - assert body["decided_at"] - - -def test_get_unknown_is_404(client: TestClient) -> None: - resp = client.get(f"/approval/requests/{uuid.uuid4()}") - assert resp.status_code == 404 - - -def test_decide_unknown_is_404(client: TestClient) -> None: - resp = client.post(f"/approval/requests/{uuid.uuid4()}/decision", json={"status": "approved"}) - assert resp.status_code == 404 diff --git a/apps/api/tests/test_integration_router.py b/apps/api/tests/test_integration_router.py index 04b188df..f1e0570f 100644 --- a/apps/api/tests/test_integration_router.py +++ b/apps/api/tests/test_integration_router.py @@ -101,9 +101,10 @@ def test_slack_notify_approval_records_ts(authenticate_app: Callable[..., FastAP """POST /slack/approvals/{id}/notify posts the gate + stashes channel/ts (AC).""" import uuid - from forge_api.routers.approval import ApprovalStore, get_approval_store from forge_api.routers.integration import ( + ApprovalStore, SlackApprovalRefStore, + get_approval_store, get_slack_notifier, get_slack_ref_store, ) @@ -137,8 +138,11 @@ def test_slack_notify_approval_404_for_unknown_gate( ) -> None: import uuid - from forge_api.routers.approval import ApprovalStore, get_approval_store - from forge_api.routers.integration import get_slack_notifier + from forge_api.routers.integration import ( + ApprovalStore, + get_approval_store, + get_slack_notifier, + ) store = ApprovalStore() slack = SlackNotifier( diff --git a/apps/api/tests/test_rbac_tenant_r2.py b/apps/api/tests/test_rbac_tenant_r2.py index 591f12d3..ba0d4f66 100644 --- a/apps/api/tests/test_rbac_tenant_r2.py +++ b/apps/api/tests/test_rbac_tenant_r2.py @@ -1,17 +1,24 @@ """Reproduction + regression tests for the Phase-2 round-2 security fixes (2.3-fix-r2). -Four real defects in the wired feature routers: +Four real defects were found in the wired feature routers. The legacy in-memory +``/approval/*`` router has since been retired in favour of the DB-backed F36 +``/approvals/*`` surface, so this file keeps the agent / workflow / spec / repo +coverage below; the approval-gate embodiment of defects #1-#3 now lives with the +DB-backed surface's own tests (``test_approvals.py``: +``test_decision_authz_matrix`` for RBAC + the ``agent-runner`` refusal, +``test_decision_body_cannot_forge_decider`` for decider identity, +``test_cross_workspace_404`` for tenant isolation). 1. **RBAC never enforced** — a read-only ``viewer`` (and the ``agent-runner`` identity) could perform writes / runs / approvals. Every write/run/approve route must now authorize, not just authenticate. -2. **HITL decider spoofing** — ``POST /approval/.../decision`` recorded - ``decided_by`` from the request body, and any role could decide. The decider - identity must come from the authenticated principal, and the ``agent-runner`` - must not be able to decide its own gate. -3. **Cross-workspace tenant isolation** — approval / agent / workflow / spec - stores were process-wide and unscoped; workspace B could read/decide/overwrite - workspace A's data. Foreign ids must surface as 404. +2. **HITL decider spoofing** — a decision recorded ``decided_by`` from the + request body, and any role could decide. The decider identity must come from + the authenticated principal, and the ``agent-runner`` must not be able to + decide its own gate. +3. **Cross-workspace tenant isolation** — agent / workflow / spec stores were + process-wide and unscoped; workspace B could read/decide/overwrite workspace + A's data. Foreign ids must surface as 404. 4. **sync_repo confused-deputy** — the route ignored ``connection_id`` and synced a caller-supplied repo with the server's privileged token. The connection must be resolved server-side, scoped to the caller's workspace. @@ -108,90 +115,32 @@ def test_viewer_cannot_write_or_run_across_routers() -> None: rid = uuid.uuid4() with TestClient(app) as client: cases = [ - client.post(f"/approval/requests/{rid}/decision", json={"status": "approved"}), client.post(f"/spec/specs/{rid}/approve"), client.post(f"/workflow/runs/{rid}/transition", json={"event": "x"}), client.post( "/integration/github/pull-requests", json={"repo": "org/api", "title": "t", "head": "f", "base": "main"}, ), - client.post( - "/approval/requests", - json={"gate": "pr", "title": "t", "confidence": 0.5}, - ), ] for resp in cases: assert resp.status_code == 403, resp.text -def test_viewer_may_still_read() -> None: - app = create_app() - _as(app, make_test_principal(role=UserRole.VIEWER)) - with TestClient(app) as client: - resp = client.get("/approval/requests") - assert resp.status_code == 200, resp.text - - # --------------------------------------------------------------------------- # -# 2. HITL decider identity # +# 2. HITL decider identity + 3. approval tenant isolation # +# # +# The approval-gate embodiment of these fixes moved to the DB-backed F36 # +# ``/approvals/*`` surface when the legacy in-memory ``/approval/*`` router was # +# retired; see ``test_approvals.py`` (``test_decision_body_cannot_forge_decider``,# +# ``test_decision_authz_matrix``, ``test_cross_workspace_404``). # # --------------------------------------------------------------------------- # -def _open_request(client: TestClient) -> str: - resp = client.post( - "/approval/requests", - json={"gate": "pr", "title": "Approve PR", "confidence": 0.8}, - ) - assert resp.status_code == 201, resp.text - return resp.json()["id"] - - -def test_decider_identity_comes_from_principal_not_body() -> None: - app = create_app() - member = make_test_principal(role=UserRole.MEMBER) - _as(app, member) - with TestClient(app) as client: - rid = _open_request(client) - resp = client.post( - f"/approval/requests/{rid}/decision", - json={"status": "approved", "decided_by": "attacker", "reason": "ok"}, - ) - assert resp.status_code == 200, resp.text - body = resp.json() - # The forged body identity is ignored; the authenticated principal is recorded. - assert body["decided_by"] != "attacker" - assert body["decided_by"] == member.email - - -def test_agent_runner_cannot_decide_its_own_gate() -> None: - app = create_app() - # Member opens the gate, then the agent-runner attempts to approve it. - _as(app, make_test_principal(role=UserRole.MEMBER)) - with TestClient(app) as client: - rid = _open_request(client) - _as(app, make_test_principal(role=UserRole.AGENT_RUNNER)) - resp = client.post(f"/approval/requests/{rid}/decision", json={"status": "approved"}) - assert resp.status_code == 403, resp.text - - # --------------------------------------------------------------------------- # -# 3. Cross-workspace tenant isolation # +# 3. Cross-workspace tenant isolation (agent / workflow / spec) # # --------------------------------------------------------------------------- # -def test_approval_is_workspace_scoped() -> None: - app = create_app() - _as(app, make_test_principal(role=UserRole.MEMBER, workspace_id=TEST_WORKSPACE_ID)) - with TestClient(app) as client: - rid = _open_request(client) - # Switch to a different tenant. - _as(app, make_test_principal(role=UserRole.MEMBER, workspace_id=OTHER_WORKSPACE_ID)) - assert client.get(f"/approval/requests/{rid}").status_code == 404 - assert client.get("/approval/requests").json() == [] - decide = client.post(f"/approval/requests/{rid}/decision", json={"status": "approved"}) - assert decide.status_code == 404 - - def test_agent_run_is_workspace_scoped() -> None: from forge_api.routers.agent import ( AgentRunStore, diff --git a/apps/api/tests/test_red_team_trigger_api.py b/apps/api/tests/test_red_team_trigger_api.py new file mode 100644 index 00000000..1ecc7fe1 --- /dev/null +++ b/apps/api/tests/test_red_team_trigger_api.py @@ -0,0 +1,252 @@ +"""Integration tests for the Red-Team Gate V1 parity + trigger endpoint (Task 20). + +Two surfaces over real handlers on hermetic SQLite (mirrors +``test_red_team_api.py`` / ``test_workflow_router.py`` conventions): + +* the V1 (FSM) gate-arrival mint — a run transitioned into ``spec_review`` + through ``POST /workflow/runs/{id}/transition`` persists ONE honest parked + verdict row (no adversary is configured in the API process), exactly where + the Temporal spine scans (post-``submit_spec_for_review``, pre-human-gate); +* ``POST /workflow/runs/{id}/red-team`` — explicit trigger: 202 + record id, + appends to the scan history, 409 off-gate, 404 unknown/foreign, 403 viewer. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import StaticPool, create_engine +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.db import get_db +from forge_api.deps import Principal, get_current_principal +from forge_api.main import create_app +from forge_api.routers.workflow import ( + WorkflowOwnership, + get_workflow_engine, + get_workflow_ownership, +) +from forge_contracts import UserRole +from forge_db.base import Base +from forge_db.models import Workspace +from forge_workflow import WorkflowEngineImpl + +WS = uuid.UUID("00000000-0000-0000-0000-0000000000e1") +WS2 = uuid.UUID("00000000-0000-0000-0000-0000000000e2") + +PARKED_EVIDENCE = {"parked": True, "reason": "no adversary model/sandbox wired"} + +#: created -> spec_drafting -> clarification -> spec_review (the human spec gate). +_TO_SPEC_REVIEW = ("generate_spec_draft", "gather_clarifications", "submit_spec_for_review") + + +@pytest.fixture +def factory() -> Iterator[sessionmaker[Session]]: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + Base.metadata.create_all(engine) + sf: sessionmaker[Session] = sessionmaker(bind=engine, expire_on_commit=False, class_=Session) + with sf() as s: + s.add(Workspace(id=WS, name="Acme", slug="acme")) + s.add(Workspace(id=WS2, name="Rival", slug="rival")) + s.commit() + yield sf + engine.dispose() + + +def _principal(workspace_id: uuid.UUID = WS, role: UserRole = UserRole.MEMBER) -> Principal: + return Principal( + user_id=uuid.uuid4(), + workspace_id=workspace_id, + role=role, + email="member@acme.test", + auth_method="test", + scopes=["*"], + ) + + +@pytest.fixture +def harness(factory: sessionmaker[Session]): + """(app, make_client) with a fresh V1 engine + ownership per test.""" + app: FastAPI = create_app() + engine = WorkflowEngineImpl() + ownership = WorkflowOwnership() + + def _get_db() -> Iterator[Session]: + with factory() as session: + yield session + + app.dependency_overrides[get_db] = _get_db + app.dependency_overrides[get_workflow_engine] = lambda: engine + app.dependency_overrides[get_workflow_ownership] = lambda: ownership + + def make_client(principal: Principal) -> TestClient: + app.dependency_overrides[get_current_principal] = lambda: principal + return TestClient(app) + + return app, make_client + + +def _start(client: TestClient) -> str: + resp = client.post("/workflow/runs", json={"task_id": str(uuid.uuid4())}) + assert resp.status_code == 201, resp.text + return resp.json()["id"] + + +def _drive(client: TestClient, run_id: str, *events: str) -> None: + for event in events: + resp = client.post(f"/workflow/runs/{run_id}/transition", json={"event": event}) + assert resp.status_code == 200, f"{event}: {resp.text}" + + +def _records(client: TestClient, run_id: str) -> dict: + resp = client.get(f"/workflow/runs/{run_id}/red-team") + assert resp.status_code == 200, resp.text + return resp.json() + + +# --------------------------------------------------------------------------- # +# V1 gate-arrival parity # +# --------------------------------------------------------------------------- # + + +def test_v1_run_reaching_spec_review_persists_parked_verdict(harness) -> None: + """A V1 run transitioned into ``spec_review`` mints one honest parked + verdict row — same shape as the Temporal default — with no extra calls.""" + _, make_client = harness + client = make_client(_principal()) + run_id = _start(client) + + _drive(client, run_id, *_TO_SPEC_REVIEW) + + body = _records(client, run_id) + assert body["latest"] is not None + assert body["latest"]["verdict"] == "survived" + assert body["latest"]["kind"] == "parked" + assert body["latest"]["evidence"] == PARKED_EVIDENCE + assert body["latest"]["adversary_model"] is None + assert body["latest"]["coder_model"] is None + assert len(body["records"]) == 1 + + +def test_v1_gate_mint_is_once_per_run(harness) -> None: + """Re-entering the gate (changes requested -> resubmit) does not rescan.""" + _, make_client = harness + client = make_client(_principal()) + run_id = _start(client) + _drive(client, run_id, *_TO_SPEC_REVIEW) + _drive(client, run_id, "spec_changes_requested", "submit_spec_for_review") + + body = _records(client, run_id) + assert len(body["records"]) == 1 + + +def test_v1_pre_gate_states_record_nothing(harness) -> None: + """No verdict is minted before the run reaches the gate.""" + _, make_client = harness + client = make_client(_principal()) + run_id = _start(client) + _drive(client, run_id, "generate_spec_draft") + + body = _records(client, run_id) + assert body["latest"] is None + assert body["records"] == [] + + +# --------------------------------------------------------------------------- # +# POST /workflow/runs/{id}/red-team (trigger) # +# --------------------------------------------------------------------------- # + + +def test_trigger_returns_202_and_get_returns_the_verdict(harness) -> None: + _, make_client = harness + client = make_client(_principal()) + run_id = _start(client) + _drive(client, run_id, *_TO_SPEC_REVIEW) + + resp = client.post(f"/workflow/runs/{run_id}/red-team") + assert resp.status_code == 202, resp.text + body = resp.json() + assert body["workflow_run_id"] == run_id + assert body["verdict"] == "survived" + assert body["kind"] == "parked" + record_id = body["record_id"] + + # NOTE: ``latest`` identity is not asserted — SQLite's second-resolution + # CURRENT_TIMESTAMP ties the gate-arrival mint with the triggered scan and + # the repository tiebreaks on ``id.desc()`` (random UUID order), a + # documented property of ``RedTeamRepository.get_by_run`` (see + # ``test_red_team_api.py``). Membership + shape is the stable contract. + got = _records(client, run_id) + triggered = [r for r in got["records"] if r["id"] == record_id] + assert len(triggered) == 1 + assert triggered[0]["verdict"] == "survived" + assert triggered[0]["kind"] == "parked" + assert triggered[0]["evidence"] == PARKED_EVIDENCE + + +def test_trigger_appends_to_the_scan_history(harness) -> None: + """The gate-arrival mint plus an explicit trigger = two records.""" + _, make_client = harness + client = make_client(_principal()) + run_id = _start(client) + _drive(client, run_id, *_TO_SPEC_REVIEW) # mints record #1 + + resp = client.post(f"/workflow/runs/{run_id}/red-team") + assert resp.status_code == 202, resp.text + + body = _records(client, run_id) + assert len(body["records"]) == 2 + assert resp.json()["record_id"] in {r["id"] for r in body["records"]} + + +def test_trigger_off_gate_is_409(harness) -> None: + """A run not at a gateable state (freshly created) conflicts.""" + _, make_client = harness + client = make_client(_principal()) + run_id = _start(client) + + resp = client.post(f"/workflow/runs/{run_id}/red-team") + assert resp.status_code == 409, resp.text + assert "gateable" in resp.json()["detail"] + + # And nothing was recorded (no silent gating either way). + assert _records(client, run_id)["records"] == [] + + +def test_trigger_unknown_run_is_404(harness) -> None: + _, make_client = harness + client = make_client(_principal()) + + resp = client.post(f"/workflow/runs/{uuid.uuid4()}/red-team") + assert resp.status_code == 404 + + +def test_trigger_foreign_workspace_run_is_404(harness) -> None: + """Another workspace's run id reads as nonexistent (no existence leak).""" + _, make_client = harness + owner_client = make_client(_principal(workspace_id=WS)) + run_id = _start(owner_client) + _drive(owner_client, run_id, *_TO_SPEC_REVIEW) + + foreign_client = make_client(_principal(workspace_id=WS2)) + resp = foreign_client.post(f"/workflow/runs/{run_id}/red-team") + assert resp.status_code == 404 + + +def test_trigger_requires_write_permission(harness) -> None: + """A read-only viewer cannot trigger a scan (matches the router's other + write endpoints).""" + _, make_client = harness + member = make_client(_principal()) + run_id = _start(member) + _drive(member, run_id, *_TO_SPEC_REVIEW) + + viewer = make_client(_principal(role=UserRole.VIEWER)) + resp = viewer.post(f"/workflow/runs/{run_id}/red-team") + assert resp.status_code == 403 diff --git a/apps/api/tests/test_slack_inbound.py b/apps/api/tests/test_slack_inbound.py index dc456505..c691a2aa 100644 --- a/apps/api/tests/test_slack_inbound.py +++ b/apps/api/tests/test_slack_inbound.py @@ -26,9 +26,10 @@ from forge_api.deps import Principal from forge_api.main import create_app from forge_api.observability.audit import AuditCategory, AuditLog -from forge_api.routers.approval import ApprovalStore, get_approval_store from forge_api.routers.integration import ( + ApprovalStore, SlackApprovalRefStore, + get_approval_store, get_integration_audit_log, get_slack_notifier, get_slack_ref_store, @@ -43,6 +44,9 @@ # across the full-suite run, where a nested tests/sso/conftest.py collides). WS_ID = uuid.UUID("00000000-0000-0000-0000-0000000000c6") USER_ID = uuid.UUID("00000000-0000-0000-0000-0000000000d7") +# A second, distinct workspace used only by the genuine cross-tenant test below — +# never the authenticated test principal's own workspace (WS_ID). +WS_ID_OTHER = uuid.UUID("00000000-0000-0000-0000-0000000000c7") def _principal() -> Principal: @@ -336,3 +340,150 @@ def test_interaction_unrecognised_action_is_ignored_200( ) assert resp.status_code == 200, resp.text assert store.get(req.id, workspace_id=WS_ID).status is ApprovalStatus.PENDING # type: ignore[union-attr, arg-type] + + +# --------------------------------------------------------------------------- # +# /forge status — wired to the live agent-run store (Task 12) # +# # +# The slash command is unauthenticated untrusted intake (no Forge principal), # +# so — mirroring the interactivity handler's ``owner_of`` resolution — it # +# resolves the owning workspace from the run id, then reads the run back # +# through the normal workspace-scoped ``AgentRunStore.get`` path. Seeds via the # +# same store the runs API tests (test_agent_router.py) drive. # +# --------------------------------------------------------------------------- # + + +def _seeded_run_store( + *, status: object | None = None, n_steps: int = 2, workspace_id: uuid.UUID = WS_ID +) -> tuple[object, object]: + """A run store seeded with one ``workspace_id``-owned run (defaults to + WS_ID; mirrors test_agent_router).""" + from forge_api.routers.agent import AgentRunStore + from forge_contracts import AgentRunResult, Step + from forge_contracts.enums import RunStatus + + store = AgentRunStore() + result = AgentRunResult( + run_id=uuid.uuid4(), + status=status if status is not None else RunStatus.SUCCEEDED, + steps=[Step(index=i) for i in range(n_steps)], + ) + store.put(result, workspace_id=workspace_id) + return store, result + + +def _status_command(text: str) -> bytes: + return urlencode({"command": "/forge", "text": text}).encode() + + +def test_status_command_returns_live_run_status( + authenticate_app: Callable[..., FastAPI], +) -> None: + from forge_api.routers.agent import get_agent_store + + store, result = _seeded_run_store(n_steps=2) + app = _build_app(authenticate_app) + app.dependency_overrides[get_agent_store] = lambda: store + body = _status_command(f"status {result.run_id}") # type: ignore[attr-defined] + with TestClient(app) as c: + resp = c.post( + "/integration/slack/commands", content=body, headers=_signed(SIGNING_SECRET, body) + ) + assert resp.status_code == 200, resp.text + text = json.dumps(resp.json()) + assert f"run {result.run_id}" in text # type: ignore[attr-defined] + assert "succeeded" in text # the run's real status string, not the old stub + assert "2 steps" in text # step detail carried by the run record + assert "not wired" not in text # the stub is gone + + +def test_status_command_unknown_run_returns_not_found( + authenticate_app: Callable[..., FastAPI], +) -> None: + from forge_api.routers.agent import get_agent_store + + store, _ = _seeded_run_store() + app = _build_app(authenticate_app) + app.dependency_overrides[get_agent_store] = lambda: store + unknown = uuid.uuid4() + body = _status_command(f"status {unknown}") + with TestClient(app) as c: + resp = c.post( + "/integration/slack/commands", content=body, headers=_signed(SIGNING_SECRET, body) + ) + assert resp.status_code == 200, resp.text + text = json.dumps(resp.json()).lower() + assert "no run" in text and "found" in text # the not-found copy + assert "succeeded" not in text # never leaks another run's status + + +def test_status_command_run_without_steps_omits_step_detail( + authenticate_app: Callable[..., FastAPI], +) -> None: + """A run with zero recorded steps renders its status with no step-count + suffix: ``_run_status_body`` only appends ``(N steps)`` when the run + actually carries any. (Previously misnamed + ``test_status_command_cross_tenant_run_is_not_found`` — it seeded and read + back a single workspace's own run, so it never exercised a second tenant; + see ``test_status_command_cross_tenant_run_is_found_by_uuid`` below for the + genuine cross-tenant case.)""" + from forge_api.routers.agent import get_agent_store + + store, result = _seeded_run_store(n_steps=0) + app = _build_app(authenticate_app) + app.dependency_overrides[get_agent_store] = lambda: store + body = _status_command(f"status {result.run_id}") # type: ignore[attr-defined] + with TestClient(app) as c: + resp = c.post( + "/integration/slack/commands", content=body, headers=_signed(SIGNING_SECRET, body) + ) + assert resp.status_code == 200, resp.text + text = json.dumps(resp.json()) + assert f"run {result.run_id}: succeeded" in text # type: ignore[attr-defined] + assert "steps" not in text # no step detail when the run carries no steps + + +def test_status_command_cross_tenant_run_is_found_by_uuid( + authenticate_app: Callable[..., FastAPI], +) -> None: + """The slash command carries no Forge tenant principal, so a run seeded + under a different workspace (WS_ID_OTHER — never the authenticated test + principal's own WS_ID) is still resolved and returned by UUID: ``owner_of`` + finds the true owning workspace and ``get`` reads the run back through + that workspace's own scope, regardless of who is asking. This is the + accepted residual posture recorded in the threat model + (docs/security/threat-model.md, S5): after signature verification, + object-UUID unguessability — not tenant matching — is the sole + cross-tenant secrecy control on this surface.""" + from forge_api.routers.agent import get_agent_store + + store, result = _seeded_run_store(n_steps=1, workspace_id=WS_ID_OTHER) + app = _build_app(authenticate_app) + app.dependency_overrides[get_agent_store] = lambda: store + body = _status_command(f"status {result.run_id}") # type: ignore[attr-defined] + with TestClient(app) as c: + resp = c.post( + "/integration/slack/commands", content=body, headers=_signed(SIGNING_SECRET, body) + ) + assert resp.status_code == 200, resp.text + text = json.dumps(resp.json()) + assert f"run {result.run_id}: succeeded" in text # type: ignore[attr-defined] + assert "no run" not in text.lower() # genuinely found, not the not-found copy + + +def test_status_command_malformed_id_returns_not_found( + authenticate_app: Callable[..., FastAPI], +) -> None: + from forge_api.routers.agent import get_agent_store + + store, _ = _seeded_run_store() + app = _build_app(authenticate_app) + app.dependency_overrides[get_agent_store] = lambda: store + body = _status_command("status not-a-uuid") + with TestClient(app) as c: + resp = c.post( + "/integration/slack/commands", content=body, headers=_signed(SIGNING_SECRET, body) + ) + assert resp.status_code == 200, resp.text + text = json.dumps(resp.json()).lower() + assert "no run" in text and "found" in text diff --git a/apps/api/tests/test_spec_router.py b/apps/api/tests/test_spec_router.py index 0c787bd6..100a3b35 100644 --- a/apps/api/tests/test_spec_router.py +++ b/apps/api/tests/test_spec_router.py @@ -221,6 +221,81 @@ def test_read_missing_spec_manifest_yaml_is_404(client: TestClient) -> None: assert resp.status_code == 404 +def test_reject_spec_persists_status_and_note(client: TestClient) -> None: + manifest = _create_spec(client) + spec_uuid = spec_id_for_key(manifest["id"]) + + resp = client.post( + f"/spec/specs/{spec_uuid}/reject", json={"note": "Missing offline handling"} + ) + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["status"] == "rejected" + assert body["review_note"] == "Missing offline handling" + + # The decision survives a re-read (persisted, not echoed). + fetched = client.get(f"/spec/specs/{spec_uuid}") + assert fetched.status_code == 200 + assert fetched.json()["status"] == "rejected" + assert fetched.json()["review_note"] == "Missing offline handling" + + +def test_request_changes_persists_status_and_note(client: TestClient) -> None: + manifest = _create_spec(client) + spec_uuid = spec_id_for_key(manifest["id"]) + + resp = client.post( + f"/spec/specs/{spec_uuid}/request-changes", json={"note": "Please add a rate limit"} + ) + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["status"] == "changes_requested" + assert body["review_note"] == "Please add a rate limit" + + fetched = client.get(f"/spec/specs/{spec_uuid}") + assert fetched.json()["status"] == "changes_requested" + assert fetched.json()["review_note"] == "Please add a rate limit" + + +def test_reject_after_approval_is_409(client: TestClient) -> None: + manifest = _create_spec(client) + spec_uuid = spec_id_for_key(manifest["id"]) + assert client.post(f"/spec/specs/{spec_uuid}/approve").status_code == 200 + + resp = client.post(f"/spec/specs/{spec_uuid}/reject", json={"note": "too late"}) + + assert resp.status_code == 409 + # The illegal transition did not mutate the spec. + assert client.get(f"/spec/specs/{spec_uuid}").json()["status"] == "approved" + + +def test_request_changes_after_approval_is_409(client: TestClient) -> None: + manifest = _create_spec(client) + spec_uuid = spec_id_for_key(manifest["id"]) + assert client.post(f"/spec/specs/{spec_uuid}/approve").status_code == 200 + + resp = client.post(f"/spec/specs/{spec_uuid}/request-changes", json={"note": "too late"}) + + assert resp.status_code == 409 + + +def test_reject_missing_spec_is_404(client: TestClient) -> None: + resp = client.post(f"/spec/specs/{uuid.uuid4()}/reject", json={"note": "no such spec"}) + assert resp.status_code == 404 + + +def test_rejected_spec_cannot_generate_tasks(client: TestClient) -> None: + manifest = _create_spec(client) + spec_uuid = spec_id_for_key(manifest["id"]) + client.post(f"/spec/specs/{spec_uuid}/reject", json={"note": "nope"}) + + resp = client.post(f"/spec/specs/{spec_uuid}/tasks") + + assert resp.status_code == 409 + + def test_lifecycle_clarify_plan_approve_tasks(client: TestClient) -> None: manifest = _create_spec(client) spec_uuid = spec_id_for_key(manifest["id"]) diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 7d3432fb..50ebfadb 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -30,10 +30,16 @@ const nextConfig = { output: "standalone", // Note: Next.js 16 removed the built-in ESLint integration (`next lint` and the // `eslint` config key). Linting now runs via `pnpm lint` (eslint.config.mjs). - // Surface the API base URL to the typed client at build time when provided. - env: { - NEXT_PUBLIC_API_URL: process.env.NEXT_PUBLIC_API_URL ?? "http://localhost:8000", - }, + // + // `NEXT_PUBLIC_API_URL` is intentionally NOT declared in an `env:` block. Next + // already inlines any `process.env.NEXT_PUBLIC_*` reference at build time from + // the build environment (a build arg wins; unset → `undefined`), exactly like + // `NEXT_PUBLIC_WS_URL`. Forcing a `?? "http://localhost:8000"` default here (as + // this block used to) would inline that literal into every bundle — a truthy, + // absolute value — so the client could never tell "unset" from "localhost", and + // the same-origin fallback in src/lib/api/api-url.ts would be dead code. Leaving + // it unset lets an un-configured build derive a same-origin `/api` base at + // runtime in the browser (see api-url.ts + docs/self-hosting/reverse-proxy.md). // HARD-09: apply the hardening headers to every route. async headers() { return [{ source: "/:path*", headers: securityHeaders }]; diff --git a/apps/web/src/app/(board)/settings/models/page.tsx b/apps/web/src/app/(board)/settings/models/page.tsx index b996ce2a..b18316bd 100644 --- a/apps/web/src/app/(board)/settings/models/page.tsx +++ b/apps/web/src/app/(board)/settings/models/page.tsx @@ -1,11 +1,21 @@ import { AoSettingsView } from "@/components/ao-settings/ao-settings-view"; +import { SelfEvalPanel } from "@/components/self-eval/self-eval-panel"; /** * Adaptive Orchestration "Models & effort" settings (`ao-settings-ui`): per-role * model + effort selectors, the tier -> model map editor, complexity thresholds, * the auto-route toggle, and a live routing-preview panel. Backed by the typed * `/ao/role-config`, `/ao/settings` and `/ao/routing-preview` routers. + * + * Below it, the Self-Eval Gate panel (`/ao/self-eval/*`): the private suite, + * frozen baseline, gate posture for pending config changes, and the run + * trigger — the gate guards exactly the config edited on this page. */ export default function AoSettingsPage() { - return ; + return ( +
+ + +
+ ); } diff --git a/apps/web/src/components/approvals/review-panel.test.tsx b/apps/web/src/components/approvals/review-panel.test.tsx index b84ea96a..e4c4cf55 100644 --- a/apps/web/src/components/approvals/review-panel.test.tsx +++ b/apps/web/src/components/approvals/review-panel.test.tsx @@ -7,6 +7,7 @@ import type { ForgeApiClient } from "@/lib/api/client"; import type { ApprovalContext, ApprovalSummary, + AttestationOut, RedTeamGateOut, } from "@/lib/api/types"; @@ -66,9 +67,28 @@ function noRedTeamScan(): Promise { return Promise.resolve({ workflow_run_id: "wf-1", latest: null, records: [] }); } +const verifiedAttestation: AttestationOut = { + id: "att-1", + changeset_hash: "sha256:" + "ab".repeat(32), + predicate_type: "https://forge.dev/attestations/changeset/v1", + keyid: "cd".repeat(32), + payload_hash: "ef".repeat(32), + created_at: "2026-07-19T00:00:00Z", + verified: true, + provenance: { + workflow_run_id: "wf-1", + agent_run_id: "ag-1", + pr_numbers: [7, 9], + spec_key: "F41", + spec_version: 2, + audit_seq: 12, + }, +}; + function makeClient(overrides: Partial = {}): ForgeApiClient { return { getWorkflowRunRedTeam: vi.fn(noRedTeamScan), + getApprovalAttestation: vi.fn(() => Promise.resolve(null)), ...overrides, } as unknown as ForgeApiClient; } @@ -198,6 +218,26 @@ describe("ReviewPanel — nine must-show items", () => { ); }); + it("surfaces a verified attested changeset on the run trace section", async () => { + const client = makeClient({ + getApprovalAttestation: vi.fn(() => Promise.resolve(verifiedAttestation)), + }); + renderPanel(fullContext, {}, client); + + const panel = await screen.findByTestId("attestation-panel"); + expect(panel).toHaveAttribute("data-state", "verified"); + expect(within(panel).getByText(/signature verified/i)).toBeInTheDocument(); + expect(client.getApprovalAttestation).toHaveBeenCalledWith("a1"); + }); + + it("shows the honest not-attested state when the server confirms absence", async () => { + renderPanel(fullContext); + + const panel = await screen.findByTestId("attestation-panel"); + expect(panel).toHaveAttribute("data-state", "absent"); + expect(within(panel).getByText(/not attested/i)).toBeInTheDocument(); + }); + it("shows no red-team badge before a scan has landed", async () => { renderPanel(fullContext); await waitFor(() => expect(screen.getByTestId("run-trace")).toBeInTheDocument()); diff --git a/apps/web/src/components/approvals/review-panel.tsx b/apps/web/src/components/approvals/review-panel.tsx index b824a71c..d8a930f4 100644 --- a/apps/web/src/components/approvals/review-panel.tsx +++ b/apps/web/src/components/approvals/review-panel.tsx @@ -13,6 +13,7 @@ import { } from "lucide-react"; import type { ReactNode } from "react"; +import { AttestationPanel } from "@/components/attestations/attestation-panel"; import { ErrorState } from "@/components/ui/error-state"; import { Loading, Skeleton } from "@/components/ui/skeleton"; import { apiClient, type ForgeApiClient } from "@/lib/api/client"; @@ -177,7 +178,7 @@ export function ReviewPanel({ {/* 8 — Run trace */} {runTrace ? (
- +
) : null} @@ -540,9 +541,11 @@ function RiskRow({ flag }: { flag: RiskFlag }) { function RunTrace({ runTrace, + approvalId, client, }: { runTrace: Record; + approvalId: string; client?: ForgeApiClient; }) { const entries = Object.entries(runTrace).filter(([, v]) => scalar(v) !== null); @@ -556,6 +559,10 @@ function RunTrace({ was blocked by) before reaching this human gate. Renders nothing until a scan has landed — see RedTeamBadge. */} + {/* Attested Changesets: the DSSE-signed provenance record for this + gate's run — verified / verification-failed / honestly absent + (records are minted on approval). See AttestationPanel. */} +
{entries.map(([key, value]) => (
diff --git a/apps/web/src/components/attestations/attestation-panel.test.tsx b/apps/web/src/components/attestations/attestation-panel.test.tsx new file mode 100644 index 00000000..44168895 --- /dev/null +++ b/apps/web/src/components/attestations/attestation-panel.test.tsx @@ -0,0 +1,112 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import type { ForgeApiClient } from "@/lib/api/client"; +import type { AttestationOut } from "@/lib/api/types"; + +import { AttestationPanel } from "./attestation-panel"; + +const verifiedAttestation: AttestationOut = { + id: "att-1", + changeset_hash: "sha256:" + "ab".repeat(32), + predicate_type: "https://forge.dev/attestations/changeset/v1", + keyid: "cd".repeat(32), + payload_hash: "ef".repeat(32), + created_at: "2026-07-19T00:00:00Z", + verified: true, + provenance: { + workflow_run_id: "wf-1", + agent_run_id: "ag-1", + pr_numbers: [7, 9], + spec_key: "F41", + spec_version: 2, + audit_seq: 12, + }, +}; + +function makeClient( + getApprovalAttestation: () => Promise, +): ForgeApiClient { + return { + getApprovalAttestation: vi.fn(getApprovalAttestation), + } as unknown as ForgeApiClient; +} + +function renderPanel(approvalId: string | null | undefined, client: ForgeApiClient) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + function Wrapper({ children }: { children: ReactNode }) { + return ( + {children} + ); + } + return render(, { + wrapper: Wrapper, + }); +} + +describe("AttestationPanel", () => { + it("renders nothing when no approval id is known", () => { + const client = makeClient(() => Promise.resolve(null)); + const { container } = renderPanel(null, client); + expect(container).toBeEmptyDOMElement(); + expect(client.getApprovalAttestation).not.toHaveBeenCalled(); + }); + + it("shows an honest absent state when no attestation exists (404)", async () => { + const client = makeClient(() => Promise.resolve(null)); + renderPanel("a1", client); + + const panel = await screen.findByTestId("attestation-panel"); + expect(panel).toHaveAttribute("data-state", "absent"); + expect(within(panel).getByText(/not attested/i)).toBeInTheDocument(); + expect(client.getApprovalAttestation).toHaveBeenCalledWith("a1"); + }); + + it("shows a verified state and reveals provenance on expand", async () => { + const client = makeClient(() => Promise.resolve(verifiedAttestation)); + renderPanel("a1", client); + + const panel = await screen.findByTestId("attestation-panel"); + expect(panel).toHaveAttribute("data-state", "verified"); + expect(within(panel).getByText(/signature verified/i)).toBeInTheDocument(); + + const toggle = within(panel).getByRole("button"); + expect(toggle).toHaveAttribute("aria-expanded", "false"); + expect(screen.queryByTestId("attestation-details")).not.toBeInTheDocument(); + + fireEvent.click(toggle); + + expect(toggle).toHaveAttribute("aria-expanded", "true"); + const details = await screen.findByTestId("attestation-details"); + expect(within(details).getByText(verifiedAttestation.changeset_hash)).toBeInTheDocument(); + expect(within(details).getByText(verifiedAttestation.keyid)).toBeInTheDocument(); + expect(within(details).getByText("7, 9")).toBeInTheDocument(); + expect(within(details).getByText("F41 v2")).toBeInTheDocument(); + }); + + it("shows an honest failure state when the signature does not verify", async () => { + const client = makeClient(() => + Promise.resolve({ ...verifiedAttestation, verified: false }), + ); + renderPanel("a1", client); + + const panel = await screen.findByTestId("attestation-panel"); + expect(panel).toHaveAttribute("data-state", "verification-failed"); + expect( + within(panel).getByText(/signature failed verification/i), + ).toBeInTheDocument(); + expect(within(panel).queryByText(/signature verified/i)).not.toBeInTheDocument(); + }); + + it("renders nothing when the fetch fails (an error is not proof of absence)", async () => { + const client = makeClient(() => Promise.reject(new Error("network down"))); + const { container } = renderPanel("a1", client); + + await waitFor(() => expect(client.getApprovalAttestation).toHaveBeenCalled()); + await waitFor(() => expect(container).toBeEmptyDOMElement()); + }); +}); diff --git a/apps/web/src/components/attestations/attestation-panel.tsx b/apps/web/src/components/attestations/attestation-panel.tsx new file mode 100644 index 00000000..476b6456 --- /dev/null +++ b/apps/web/src/components/attestations/attestation-panel.tsx @@ -0,0 +1,131 @@ +"use client"; + +import { BadgeCheck, BadgeX, FileQuestion } from "lucide-react"; +import { useState } from "react"; + +import { useApprovalAttestation } from "@/lib/api/approvals"; +import { apiClient, type ForgeApiClient } from "@/lib/api/client"; +import type { AttestationOut } from "@/lib/api/types"; +import { cn } from "@/lib/utils"; + +export interface AttestationPanelProps { + /** + * The approval gate whose attestation to show (`ApprovalSummary.id`). + * Absent -> nothing renders. + */ + approvalId?: string | null; + client?: ForgeApiClient; +} + +/** + * The Attested Changeset panel (Attested Changesets, Task 19). + * + * Approving a `pr` gate that carries a workflow run mints a DSSE/Ed25519-signed + * provenance record over the changeset. This panel shows exactly what the + * server can vouch for — three honest states, no fake ones: + * + * - **verified** — a signed record exists and its signature verifies against + * the deployment's key (computed server-side by the same path the + * `forge-verify` CLI uses); + * - **verification-failed** — a signed record exists but its signature does + * NOT verify (wrong or rotated key, or a tampered record); + * - **absent** — the server confirmed no attestation exists (normal while the + * gate is still pending: records are minted on approval). + * + * While loading — or when the fetch itself fails — nothing renders: a failed + * request is not proof of absence, so the panel stays quiet rather than + * claiming a state it cannot back. + */ +export function AttestationPanel({ approvalId, client = apiClient }: AttestationPanelProps) { + const [expanded, setExpanded] = useState(false); + const query = useApprovalAttestation(approvalId, client); + + if (!approvalId || query.isPending || query.isError) { + return null; + } + + const attestation = query.data ?? null; + + if (attestation === null) { + return ( +
+ + Not attested — no signed changeset record for this gate yet +
+ ); + } + + const failed = !attestation.verified; + const Icon = failed ? BadgeX : BadgeCheck; + + return ( +
+ + + {expanded ? : null} +
+ ); +} + +function DetailRow({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function AttestationDetails({ attestation }: { attestation: AttestationOut }) { + const { provenance } = attestation; + const spec = + provenance.spec_key && provenance.spec_version + ? `${provenance.spec_key} v${provenance.spec_version}` + : null; + return ( +
+
+ + + {provenance.pr_numbers.length > 0 ? ( + + ) : null} + {spec ? : null} + {provenance.audit_seq != null ? ( + + ) : null} + +
+
+ ); +} diff --git a/apps/web/src/components/self-eval/self-eval-panel.test.tsx b/apps/web/src/components/self-eval/self-eval-panel.test.tsx new file mode 100644 index 00000000..490419fd --- /dev/null +++ b/apps/web/src/components/self-eval/self-eval-panel.test.tsx @@ -0,0 +1,197 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { ApiError, type ForgeApiClient } from "@/lib/api/client"; +import type { SelfEvalRunAccepted, SelfEvalStatusOut } from "@/lib/api/types"; + +import { SelfEvalPanel } from "./self-eval-panel"; + +function makeStatus(over: Partial = {}): SelfEvalStatusOut { + return { + workspace_id: "w-test", + enforced: true, + suite: { + id: "s-1", + slug: "acme-app-self-eval", + version: "1.2.0", + title: "Acme private suite", + task_count: 12, + repo_id: "github:acme/app", + published: true, + }, + baseline: { + benchmark_suite_id: "s-1", + baseline_rate: 0.8, + resolved: 8, + total: 10, + recorded_at: "2026-07-01T12:00:00Z", + }, + ...over, + }; +} + +function makeAccepted(): SelfEvalRunAccepted { + return { + status: "queued", + task: "forge.self_eval.run", + workspace_id: "w-test", + benchmark_suite_id: "s-1", + }; +} + +function makeClient(overrides: Partial = {}): ForgeApiClient { + return { + getSelfEvalStatus: vi.fn(() => Promise.resolve(makeStatus())), + runSelfEval: vi.fn(() => Promise.resolve(makeAccepted())), + ...overrides, + } as unknown as ForgeApiClient; +} + +function renderPanel(client: ForgeApiClient) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); + function Wrapper({ children }: { children: ReactNode }) { + return ( + {children} + ); + } + return render(, { wrapper: Wrapper }); +} + +describe("SelfEvalPanel", () => { + it("renders the loading skeleton while the status loads", () => { + renderPanel( + makeClient({ + getSelfEvalStatus: vi.fn(() => new Promise(() => {})), + }), + ); + expect(screen.getByTestId("self-eval-skeleton")).toBeInTheDocument(); + }); + + it("shows a fetch error state distinct from the empty (cold-start) state", async () => { + renderPanel( + makeClient({ + getSelfEvalStatus: vi.fn(() => + Promise.reject(new ApiError(500, "boom", null)), + ), + }), + ); + expect(await screen.findByTestId("self-eval-error")).toBeInTheDocument(); + expect(screen.queryByTestId("self-eval-panel")).not.toBeInTheDocument(); + expect(screen.queryByTestId("self-eval-no-baseline")).not.toBeInTheDocument(); + }); + + it("renders suite, baseline and last-run facts from the API", async () => { + renderPanel(makeClient()); + + expect(await screen.findByTestId("self-eval-panel")).toBeInTheDocument(); + expect(screen.getByTestId("self-eval-suite")).toHaveTextContent( + "acme-app-self-eval", + ); + expect(screen.getByTestId("self-eval-suite")).toHaveTextContent("1.2.0"); + expect(screen.getByTestId("self-eval-suite")).toHaveTextContent( + "github:acme/app", + ); + expect(screen.getByTestId("self-eval-baseline")).toHaveTextContent("80.0%"); + expect(screen.getByTestId("self-eval-baseline")).toHaveTextContent("8/10"); + expect(screen.getByTestId("self-eval-last-run")).toHaveTextContent("8/10"); + }); + + it("shows the gate as able to block when enforcement is on and a baseline exists", async () => { + renderPanel(makeClient()); + const gate = await screen.findByTestId("self-eval-gate-status"); + expect(gate).toHaveTextContent(/enforcement on/i); + // The Phase-A limitation is stated inline, always. + expect(screen.getByTestId("self-eval-phase-a")).toHaveTextContent(/phase a/i); + }); + + it("shows the gate as off when enforcement is disabled", async () => { + renderPanel( + makeClient({ + getSelfEvalStatus: vi.fn(() => + Promise.resolve(makeStatus({ enforced: false })), + ), + }), + ); + expect(await screen.findByTestId("self-eval-gate-status")).toHaveTextContent( + /enforcement off/i, + ); + }); + + it("states plainly that the gate cannot block until a baseline exists", async () => { + renderPanel( + makeClient({ + getSelfEvalStatus: vi.fn(() => + Promise.resolve(makeStatus({ baseline: null })), + ), + }), + ); + const empty = await screen.findByTestId("self-eval-no-baseline"); + expect(empty).toHaveTextContent(/cannot block any config change/i); + expect(screen.getByTestId("self-eval-last-run")).toHaveTextContent( + /no scored runs/i, + ); + }); + + it("explains the missing suite on cold start and disables the run action", async () => { + renderPanel( + makeClient({ + getSelfEvalStatus: vi.fn(() => + Promise.resolve(makeStatus({ suite: null, baseline: null })), + ), + }), + ); + expect(await screen.findByTestId("self-eval-no-suite")).toHaveTextContent( + /no private suite/i, + ); + expect(screen.getByTestId("self-eval-run")).toBeDisabled(); + }); + + it("disables the run action for an unpublished suite", async () => { + const status = makeStatus(); + renderPanel( + makeClient({ + getSelfEvalStatus: vi.fn(() => + Promise.resolve({ + ...status, + suite: { ...status.suite!, published: false }, + }), + ), + }), + ); + await screen.findByTestId("self-eval-panel"); + expect(screen.getByTestId("self-eval-run")).toBeDisabled(); + }); + + it("queues a run and renders the accepted state", async () => { + const client = makeClient(); + renderPanel(client); + await screen.findByTestId("self-eval-panel"); + + fireEvent.click(screen.getByTestId("self-eval-run")); + + await waitFor(() => expect(client.runSelfEval).toHaveBeenCalledTimes(1)); + expect(await screen.findByTestId("self-eval-run-accepted")).toHaveTextContent( + /queued/i, + ); + }); + + it("shows a run error distinct from the accepted state", async () => { + renderPanel( + makeClient({ + runSelfEval: vi.fn(() => + Promise.reject(new ApiError(409, "no_private_suite", null)), + ), + }), + ); + await screen.findByTestId("self-eval-panel"); + + fireEvent.click(screen.getByTestId("self-eval-run")); + + expect(await screen.findByTestId("self-eval-run-error")).toBeInTheDocument(); + expect(screen.queryByTestId("self-eval-run-accepted")).not.toBeInTheDocument(); + }); +}); diff --git a/apps/web/src/components/self-eval/self-eval-panel.tsx b/apps/web/src/components/self-eval/self-eval-panel.tsx new file mode 100644 index 00000000..a0731d1b --- /dev/null +++ b/apps/web/src/components/self-eval/self-eval-panel.tsx @@ -0,0 +1,327 @@ +"use client"; + +import { FlaskConical, Play, ShieldCheck } from "lucide-react"; +import type { ReactNode } from "react"; + +import { apiClient, type ForgeApiClient } from "@/lib/api/client"; +import { useRunSelfEval, useSelfEvalStatus } from "@/lib/api/ao-settings"; +import type { SelfEvalStatusOut } from "@/lib/api/types"; +import { cn } from "@/lib/utils"; + +export interface SelfEvalPanelProps { + client?: ForgeApiClient; +} + +/** + * Self-Eval Gate settings panel (trust layer, Phase A): the workspace's + * private per-repo suite, the frozen baseline the gate defends, the last + * scoring run, the gate's current posture for pending config changes, and a + * "run self-eval" action that enqueues the worker-owned `forge.self_eval.run` + * task via `POST /ao/self-eval/runs`. Every state is backed by + * `GET /ao/self-eval/status`; the Phase-A limitation is stated inline rather + * than hidden. + */ +export function SelfEvalPanel({ client = apiClient }: SelfEvalPanelProps) { + const statusQuery = useSelfEvalStatus(client); + const runMutation = useRunSelfEval(client); + + if (statusQuery.isLoading) { + return ( +
+
+
+ ); + } + + if (statusQuery.isError || !statusQuery.data) { + return ( +
+ +
+

+ Self-Eval Gate status unavailable +

+

+ The status read failed — this is a fetch error, not an empty + workspace. Try again in a moment. +

+
+ +
+ ); + } + + const status = statusQuery.data; + const runnable = status.suite !== null && status.suite.published; + + return ( +
+
+ + + +
+

+ Self-Eval Gate +

+

+ Blocks a model/router config change that regresses your private + per-repo regression suite below its frozen baseline. +

+
+
+ +
+ + + +
+ + + + {/* Run action — enqueues the worker task; a run is minutes-long. */} +
+
+ + {!runnable ? ( + + Requires a published private suite. + + ) : null} +
+ {runMutation.isSuccess ? ( +

+ Run queued — the forge.self_eval.run worker task will + attempt to score the private suite; if it can score, it records the + baseline. Refresh in a few minutes. +

+ ) : null} + {runMutation.isError ? ( +

+ Couldn't queue the self-eval run. It needs a published private + suite and admin access — please try again. +

+ ) : null} +
+ + {/* Phase-A limitation, stated inline (honesty rule). */} +

+ Phase A limitation:{" "} + the API layer does not re-evaluate a proposed config inline. Without a + baseline the gate cannot block anything, and even with one, a stock + deployment no-ops at config-change time until an eval runner is + injected — baselines are recorded only by the worker-owned{" "} + forge.self_eval.run task queued above. +

+
+ ); +} + +// --- Blocks ------------------------------------------------------------------ // + +function SuiteBlock({ status }: { status: SelfEvalStatusOut }) { + return ( + + {status.suite ? ( +
+ + {status.suite.slug} @ {status.suite.version} + + + {status.suite.title} · {status.suite.task_count} hidden cases + + {status.suite.repo_id ? ( + + {status.suite.repo_id} + + ) : null} + + {status.suite.published ? "published" : "unpublished — not runnable"} + +
+ ) : ( +

+ No private suite yet — one is minted from your merged PRs by the{" "} + forge.self_eval.mint worker task. +

+ )} + + ); +} + +function BaselineBlock({ status }: { status: SelfEvalStatusOut }) { + return ( + + {status.baseline ? ( +
+ + {formatRate(status.baseline.baseline_rate)} + + + {status.baseline.resolved}/{status.baseline.total} cases resolved + + + recorded {formatWhen(status.baseline.recorded_at)} + +
+ ) : ( +

+ No baseline recorded — the Self-Eval Gate cannot block any config + change until a baseline exists. +

+ )} + + ); +} + +function GateStatusBlock({ status }: { status: SelfEvalStatusOut }) { + const gate = deriveGate(status); + return ( + +
+ + {gate.label} + +

{gate.description}

+
+ + ); +} + +function LastRunLine({ status }: { status: SelfEvalStatusOut }) { + return ( +

+ + {status.baseline ? ( + <> + Last scoring run: {status.baseline.resolved}/{status.baseline.total}{" "} + resolved ({formatRate(status.baseline.baseline_rate)}) on{" "} + {formatWhen(status.baseline.recorded_at)}. + + ) : ( + <>No scored runs recorded. + )}{" "} + Phase A keeps no separate run history — a run that scores updates the + baseline; a run that cannot score (missing provisioning) leaves no record + here. +

+ ); +} + +// --- Derivations & primitives ------------------------------------------------ // + +function deriveGate(status: SelfEvalStatusOut): { + label: string; + description: string; + tone: string; +} { + if (!status.enforced) { + return { + label: "Enforcement off", + description: + "self_eval_enforce is disabled — pending config changes are not checked against the baseline.", + tone: "border-border bg-muted text-muted-foreground", + }; + } + if (!status.baseline) { + return { + label: "Enforcement on — cannot block yet", + description: + "Enforcement is on, but with no baseline there is nothing to regress against, so every pending config change passes.", + tone: "border-warning/40 bg-warning/10 text-warning", + }; + } + return { + label: "Enforcement on — baseline recorded", + description: + "A pending config change that regresses below the baseline is refused (409) unless forced — once an eval runner is wired (see the Phase A note).", + tone: "border-success/40 bg-success/10 text-success", + }; +} + +function formatRate(rate: number): string { + return `${(rate * 100).toFixed(1)}%`; +} + +function formatWhen(iso: string): string { + const date = new Date(iso); + return Number.isNaN(date.getTime()) ? iso : date.toLocaleString(); +} + +function FactBlock({ + title, + testId, + children, +}: { + title: string; + testId: string; + children: ReactNode; +}) { + return ( +
+

+ {title} +

+ {children} +
+ ); +} + +function Mono({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/apps/web/src/components/spec-studio/read-mode.test.tsx b/apps/web/src/components/spec-studio/read-mode.test.tsx index 4bff4d70..cb1938eb 100644 --- a/apps/web/src/components/spec-studio/read-mode.test.tsx +++ b/apps/web/src/components/spec-studio/read-mode.test.tsx @@ -22,6 +22,8 @@ function setup(overrides: Partial> = {}) { const props = { spec: baseSpec, onApprove: vi.fn(), + onReject: vi.fn(), + onRequestChanges: vi.fn(), ...overrides, } as React.ComponentProps; render(); @@ -65,7 +67,7 @@ describe("ReadMode", () => { expect(onApprove).toHaveBeenCalledTimes(1); }); - it("pressing 'x' opens the reject note composer, and confirming records + calls onReject", () => { + it("pressing 'x' opens the reject note composer, and confirming sends the note to onReject", () => { const onReject = vi.fn(); setup({ onReject }); fireEvent.keyDown(screen.getByTestId("read-mode"), { key: "x" }); @@ -75,11 +77,12 @@ describe("ReadMode", () => { }); fireEvent.click(screen.getByTestId("confirm-decision")); expect(onReject).toHaveBeenCalledWith("Missing offline handling"); - expect(screen.getByTestId("review-recorded")).toHaveTextContent("Rejected"); - expect(screen.getByTestId("review-recorded")).toHaveTextContent("Missing offline handling"); + // The composer closes; the decision renders from the server-persisted + // manifest (via the spec prop), never from local component state. + expect(screen.queryByTestId("reason-composer")).not.toBeInTheDocument(); }); - it("pressing 'r' opens the request-changes note composer, and confirming records + calls onRequestChanges", () => { + it("pressing 'r' opens the request-changes note composer, and confirming sends the note to onRequestChanges", () => { const onRequestChanges = vi.fn(); setup({ onRequestChanges }); fireEvent.keyDown(screen.getByTestId("read-mode"), { key: "r" }); @@ -89,10 +92,9 @@ describe("ReadMode", () => { }); fireEvent.click(screen.getByTestId("confirm-decision")); expect(onRequestChanges).toHaveBeenCalledWith("Please add a rate limit"); - expect(screen.getByTestId("review-recorded")).toHaveTextContent("Changes requested"); }); - it("Escape cancels the note composer without recording a decision", () => { + it("Escape cancels the note composer without sending a decision", () => { const onReject = vi.fn(); setup({ onReject }); fireEvent.keyDown(screen.getByTestId("read-mode"), { key: "x" }); @@ -101,7 +103,30 @@ describe("ReadMode", () => { }); expect(screen.queryByTestId("reason-composer")).not.toBeInTheDocument(); expect(onReject).not.toHaveBeenCalled(); - expect(screen.queryByTestId("review-recorded")).not.toBeInTheDocument(); + }); + + it("renders a persisted rejected decision (status + note) from the manifest", () => { + setup({ + spec: { ...baseSpec, status: "rejected", review_note: "Missing offline handling" }, + }); + expect(screen.getByTestId("read-status")).toHaveTextContent("Rejected"); + expect(screen.getByTestId("review-decision")).toHaveTextContent("Rejected"); + expect(screen.getByTestId("review-decision")).toHaveTextContent("Missing offline handling"); + }); + + it("renders a persisted changes-requested decision from the manifest", () => { + setup({ + spec: { ...baseSpec, status: "changes_requested", review_note: "Please add a rate limit" }, + }); + expect(screen.getByTestId("read-status")).toHaveTextContent("Changes requested"); + expect(screen.getByTestId("review-decision")).toHaveTextContent("Changes requested"); + expect(screen.getByTestId("review-decision")).toHaveTextContent("Please add a rate limit"); + }); + + it("keeps the review gate open for a rejected spec (the decision can be revised)", () => { + setup({ spec: { ...baseSpec, status: "rejected" } }); + expect(screen.queryByTestId("review-gate-closed")).not.toBeInTheDocument(); + expect(screen.getByTestId("decision-approve")).toBeEnabled(); }); it("disables the decision bar once the spec is past the human gate", () => { @@ -117,13 +142,13 @@ describe("ReadMode", () => { expect(onApprove).not.toHaveBeenCalled(); }); - it("surfaces an approve error from the caller", () => { - setup({ approveError: "Couldn't reach the spec engine" }); + it("surfaces a server error from any review decision", () => { + setup({ errorMessage: "Couldn't reach the spec engine" }); expect(screen.getByRole("alert")).toHaveTextContent("Couldn't reach the spec engine"); }); - it("shows a saving state on Approve while the mutation is pending", () => { - setup({ approving: true }); + it("disables the decision bar while a review decision is in flight", () => { + setup({ pending: true }); expect(screen.getByTestId("decision-approve")).toBeDisabled(); }); }); diff --git a/apps/web/src/components/spec-studio/read-mode.tsx b/apps/web/src/components/spec-studio/read-mode.tsx index c9ec5b3c..39e6dcf1 100644 --- a/apps/web/src/components/spec-studio/read-mode.tsx +++ b/apps/web/src/components/spec-studio/read-mode.tsx @@ -20,21 +20,16 @@ function isEditableTarget(target: EventTarget | null): boolean { export interface ReadModeProps { spec: SpecManifest; - /** Approves the spec at the human gate (`POST /spec/specs/{id}/approve`, real). */ + /** Approves the spec at the human gate (`POST /spec/specs/{id}/approve`). */ onApprove: () => void; - approving?: boolean; - approveError?: string | null; - /** - * Reject / request-changes have no backend endpoint yet — `forge_spec`'s - * `FileSpecEngine` only exposes `approve_spec` (no `reject_spec` / - * `request_changes` state transition, and `SpecStatus` has no such values). - * Read mode still records the decision + note locally (so the keyboard-first - * review flow is fully usable) and surfaces it through these optional - * callbacks for a caller to persist once that endpoint exists — parked, - * tracked separately; not faked as a server round-trip. - */ - onReject?: (note: string) => void; - onRequestChanges?: (note: string) => void; + /** Rejects the spec, persisting the note (`POST /spec/specs/{id}/reject`). */ + onReject: (note: string) => void; + /** Requests changes, persisting the note (`POST /spec/specs/{id}/request-changes`). */ + onRequestChanges: (note: string) => void; + /** True while any review decision (approve/reject/request-changes) is in flight. */ + pending?: boolean; + /** Server error from the most recent review decision — always surfaced, never swallowed. */ + errorMessage?: string | null; } /** @@ -49,18 +44,19 @@ export interface ReadModeProps { export function ReadMode({ spec, onApprove, - approving = false, - approveError = null, onReject, onRequestChanges, + pending = false, + errorMessage = null, }: ReadModeProps) { const [activeNote, setActiveNote] = useState<"reject" | "request_changes" | null>(null); const [note, setNote] = useState(""); - const [recorded, setRecorded] = useState<{ action: "reject" | "request_changes"; note: string } | null>( - null, - ); const reviewable = isApprovable(spec.status); + // The persisted review decision (server state, via the manifest) — never a + // local echo of a click that might not have survived the round-trip. + const decision = + spec.status === "rejected" || spec.status === "changes_requested" ? spec.status : null; const submit = useCallback( (action: ApprovalAction, reason?: string) => { @@ -68,20 +64,18 @@ export function ReadMode({ onApprove(); return; } - const decision = action as "reject" | "request_changes"; const trimmed = (reason ?? "").trim(); - setRecorded({ action: decision, note: trimmed }); setActiveNote(null); setNote(""); - if (decision === "reject") onReject?.(trimmed); - else onRequestChanges?.(trimmed); + if (action === "reject") onReject(trimmed); + else onRequestChanges(trimmed); }, [onApprove, onReject, onRequestChanges], ); const trigger = useCallback( (action: ApprovalAction) => { - if (!reviewable || approving) return; + if (!reviewable || pending) return; if (action === "reject" || action === "request_changes") { setNote(""); setActiveNote(action); @@ -89,7 +83,7 @@ export function ReadMode({ submit(action); } }, - [reviewable, approving, submit], + [reviewable, pending, submit], ); const onKeyDown = (event: KeyboardEvent) => { @@ -246,15 +240,14 @@ export function ReadMode({ ) : null}
- {recorded ? ( + {decision ? (

- {recorded.action === "reject" ? "Rejected" : "Changes requested"} - {recorded.note ? ` — "${recorded.note}"` : ""} (recorded locally; not yet - persisted server-side). + {decision === "rejected" ? "Rejected" : "Changes requested"} + {spec.review_note ? ` — "${spec.review_note}"` : ""}

) : null} activeNote && submit(activeNote, note)} onCancel={() => { diff --git a/apps/web/src/components/spec-studio/spec-studio.test.tsx b/apps/web/src/components/spec-studio/spec-studio.test.tsx index f1614b63..6929dc19 100644 --- a/apps/web/src/components/spec-studio/spec-studio.test.tsx +++ b/apps/web/src/components/spec-studio/spec-studio.test.tsx @@ -168,6 +168,119 @@ describe("SpecStudio", () => { ); }); + it("rejecting in Read mode posts the note to the server and renders the returned status", async () => { + const rejected: SpecManifest = { + ...manifest, + status: "rejected", + review_note: "Missing offline handling", + }; + const client = makeClient({ + getSpecManifest: vi.fn().mockResolvedValueOnce(manifest).mockResolvedValue(rejected), + rejectSpec: vi.fn(() => Promise.resolve(rejected)), + }); + renderStudio(client); + await screen.findByTestId("guided-mode"); + + fireEvent.click(screen.getByTestId("studio-mode-read")); + await screen.findByTestId("read-mode"); + fireEvent.click(screen.getByTestId("decision-reject")); + const composer = screen.getByTestId("reason-composer"); + fireEvent.change(composer.querySelector("textarea") as HTMLTextAreaElement, { + target: { value: "Missing offline handling" }, + }); + fireEvent.click(screen.getByTestId("confirm-decision")); + + await waitFor(() => + expect(client.rejectSpec).toHaveBeenCalledWith("SPEC-1", "Missing offline handling"), + ); + await waitFor(() => + expect(screen.getByTestId("read-status")).toHaveTextContent("Rejected"), + ); + expect(screen.getByTestId("review-decision")).toHaveTextContent("Missing offline handling"); + }); + + it("requesting changes in Read mode posts the note to the server", async () => { + const changed: SpecManifest = { + ...manifest, + status: "changes_requested", + review_note: "Please add a rate limit", + }; + const client = makeClient({ + getSpecManifest: vi.fn().mockResolvedValueOnce(manifest).mockResolvedValue(changed), + requestSpecChanges: vi.fn(() => Promise.resolve(changed)), + }); + renderStudio(client); + await screen.findByTestId("guided-mode"); + + fireEvent.click(screen.getByTestId("studio-mode-read")); + await screen.findByTestId("read-mode"); + fireEvent.click(screen.getByTestId("decision-request_changes")); + const composer = screen.getByTestId("reason-composer"); + fireEvent.change(composer.querySelector("textarea") as HTMLTextAreaElement, { + target: { value: "Please add a rate limit" }, + }); + fireEvent.click(screen.getByTestId("confirm-decision")); + + await waitFor(() => + expect(client.requestSpecChanges).toHaveBeenCalledWith("SPEC-1", "Please add a rate limit"), + ); + await waitFor(() => + expect(screen.getByTestId("read-status")).toHaveTextContent("Changes requested"), + ); + }); + + it("surfaces a server error when rejecting fails (no silent local state)", async () => { + const client = makeClient({ + rejectSpec: vi.fn(() => Promise.reject(new Error("spec engine unavailable"))), + }); + renderStudio(client); + await screen.findByTestId("guided-mode"); + + fireEvent.click(screen.getByTestId("studio-mode-read")); + await screen.findByTestId("read-mode"); + fireEvent.click(screen.getByTestId("decision-reject")); + fireEvent.change( + screen.getByTestId("reason-composer").querySelector("textarea") as HTMLTextAreaElement, + { target: { value: "nope" } }, + ); + fireEvent.click(screen.getByTestId("confirm-decision")); + + expect(await screen.findByRole("alert")).toHaveTextContent("spec engine unavailable"); + // The status must not pretend the decision persisted. + expect(screen.getByTestId("read-status")).toHaveTextContent("Draft"); + }); + + it("clears a stale rejection error once a later Approve succeeds (no unclearable alert)", async () => { + const client = makeClient({ + rejectSpec: vi.fn(() => Promise.reject(new Error("spec engine unavailable"))), + approveSpec: vi.fn(() => + Promise.resolve({ ...manifest, status: "approved" } satisfies SpecManifest), + ), + }); + renderStudio(client); + await screen.findByTestId("guided-mode"); + + fireEvent.click(screen.getByTestId("studio-mode-read")); + await screen.findByTestId("read-mode"); + + // Reject fails first — its mutation is left in an error state. + fireEvent.click(screen.getByTestId("decision-reject")); + fireEvent.change( + screen.getByTestId("reason-composer").querySelector("textarea") as HTMLTextAreaElement, + { target: { value: "nope" } }, + ); + fireEvent.click(screen.getByTestId("confirm-decision")); + expect(await screen.findByRole("alert")).toHaveTextContent("spec engine unavailable"); + + // Approve then succeeds. A `rejectSpec` mutation only clears `isError` when + // IT is re-fired, so a naive "priority ternary over all three mutations" + // would keep showing the stale reject error forever. + fireEvent.click(screen.getByTestId("decision-approve")); + await waitFor(() => expect(client.approveSpec).toHaveBeenCalledWith("SPEC-1")); + + await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); + }); + it("disables the YAML save button and surfaces errors for an invalid manifest", async () => { renderStudio(makeClient()); await screen.findByTestId("guided-mode"); diff --git a/apps/web/src/components/spec-studio/spec-studio.tsx b/apps/web/src/components/spec-studio/spec-studio.tsx index c92c1f93..bdc26e25 100644 --- a/apps/web/src/components/spec-studio/spec-studio.tsx +++ b/apps/web/src/components/spec-studio/spec-studio.tsx @@ -5,7 +5,7 @@ import { useState } from "react"; import { toast } from "@/components/ui/toast"; import { apiClient, ApiError, type ForgeApiClient } from "@/lib/api/client"; -import { useApproveSpec } from "@/lib/api/spec"; +import { useApproveSpec, useRejectSpec, useRequestSpecChanges } from "@/lib/api/spec"; import { useSaveGuidedManifest, useSaveSpecMarkdown, @@ -123,6 +123,25 @@ export function SpecStudio({ specId, client = apiClient, collab }: SpecStudioPro const saveMarkdown = useSaveSpecMarkdown(specId, client); const saveYaml = useSaveSpecManifestYaml(specId, client); const approveSpec = useApproveSpec(client); + const rejectSpec = useRejectSpec(client); + const requestChanges = useRequestSpecChanges(client); + + const reviewPending = + approveSpec.isPending || rejectSpec.isPending || requestChanges.isPending; + // A priority ternary over three independent mutations would be unsafe on + // its own: react-query keeps `isError` set on a mutation until IT is + // re-fired, so an old failure could otherwise linger in `reviewError` + // after a *different* review decision has since succeeded. Safe here only + // because each `onApprove`/`onReject`/`onRequestChanges` handler below + // resets its siblings immediately before mutating, so at most one of these + // three can be `isError` at a time. + const reviewError = approveSpec.isError + ? errorMessage(approveSpec.error) + : rejectSpec.isError + ? errorMessage(rejectSpec.error) + : requestChanges.isError + ? errorMessage(requestChanges.error) + : null; const manifest = manifestQuery.data ?? null; const guidedValue = guidedOverride ?? manifest; @@ -284,14 +303,37 @@ export function SpecStudio({ specId, client = apiClient, collab }: SpecStudioPro {mode === "read" ? ( + onApprove={() => { + // Clear the siblings' settled state *before* firing this + // mutation: a mutation keeps `isError`/`isSuccess` until IT is + // re-fired, so without this a stale error from a previous + // reject/request-changes attempt would linger in `reviewError` + // forever, even after this decision succeeds. + rejectSpec.reset(); + requestChanges.reset(); approveSpec.mutate( { specId }, { onSuccess: () => toast.success("Spec approved") }, - ) - } - approving={approveSpec.isPending} - approveError={approveSpec.isError ? errorMessage(approveSpec.error) : null} + ); + }} + onReject={(note) => { + approveSpec.reset(); + requestChanges.reset(); + rejectSpec.mutate( + { specId, note }, + { onSuccess: () => toast.success("Spec rejected") }, + ); + }} + onRequestChanges={(note) => { + approveSpec.reset(); + rejectSpec.reset(); + requestChanges.mutate( + { specId, note }, + { onSuccess: () => toast.success("Changes requested") }, + ); + }} + pending={reviewPending} + errorMessage={reviewError} /> ) : null} {mode === "history" ? : null} diff --git a/apps/web/src/components/spec/spec-meta.test.ts b/apps/web/src/components/spec/spec-meta.test.ts index 4d5a6c0b..4c535174 100644 --- a/apps/web/src/components/spec/spec-meta.test.ts +++ b/apps/web/src/components/spec/spec-meta.test.ts @@ -11,6 +11,8 @@ import { plainCurrentStep, plainStepCompletion, plainStepState, + STATUS_LABELS, + statusBadgeClass, traceSealed, } from "./spec-meta"; @@ -38,6 +40,26 @@ describe("isApprovable", () => { expect(isApprovable("approved")).toBe(false); expect(isApprovable("validated")).toBe(false); }); + + it("keeps rejected / changes-requested specs reviewable (still before the gate)", () => { + expect(isApprovable("rejected")).toBe(true); + expect(isApprovable("changes_requested")).toBe(true); + }); +}); + +describe("review decision statuses", () => { + it("labels and badges the review statuses", () => { + expect(STATUS_LABELS.rejected).toBe("Rejected"); + expect(STATUS_LABELS.changes_requested).toBe("Changes requested"); + expect(statusBadgeClass("rejected")).toContain("danger"); + expect(statusBadgeClass("changes_requested")).toContain("warning"); + }); + + it("does not count a rejected spec as past the Approve step on the lifecycle rail", () => { + const [describeDone, , approveDone] = plainStepCompletion({ status: "rejected" }); + expect(describeDone).toBe(true); // requirements were captured and reviewed + expect(approveDone).toBe(false); // the human gate was NOT passed + }); }); describe("plain-language lifecycle stepper", () => { diff --git a/apps/web/src/components/spec/spec-meta.ts b/apps/web/src/components/spec/spec-meta.ts index 1b8450fb..714105c5 100644 --- a/apps/web/src/components/spec/spec-meta.ts +++ b/apps/web/src/components/spec/spec-meta.ts @@ -19,6 +19,8 @@ export type StageState = "done" | "current" | "upcoming"; export const STATUS_LABELS: Record = { draft: "Draft", clarifying: "Clarifying", + changes_requested: "Changes requested", + rejected: "Rejected", approved: "Approved", implementing: "Implementing", validated: "Validated", @@ -35,16 +37,28 @@ export function statusBadgeClass(status: SpecStatus | undefined): string { case "implementing": return "border-primary/40 bg-primary/10 text-primary"; case "clarifying": + case "changes_requested": return "border-warning/40 bg-warning/10 text-warning"; + case "rejected": + return "border-danger/40 bg-danger/10 text-danger"; case "draft": default: return "border-border bg-muted text-muted-foreground"; } } -/** The two statuses at (or before) the human approval gate. */ +/** + * The statuses at (or before) the human approval gate. Rejected / + * changes-requested specs stay reviewable — the decision can be revised + * (mirroring the engine's `REVIEWABLE_STATUSES` gate). + */ export function isApprovable(status: SpecStatus | undefined): boolean { - return status === "draft" || status === "clarifying"; + return ( + status === "draft" || + status === "clarifying" || + status === "changes_requested" || + status === "rejected" + ); } export interface GateSummary { diff --git a/apps/web/src/components/sso/sso-settings-view.test.tsx b/apps/web/src/components/sso/sso-settings-view.test.tsx index 9ac11333..cf8467ca 100644 --- a/apps/web/src/components/sso/sso-settings-view.test.tsx +++ b/apps/web/src/components/sso/sso-settings-view.test.tsx @@ -255,6 +255,87 @@ describe("SsoSettingsView", () => { ); }); + it("renders the SLO URL field disabled with a not-yet-supported hint, but still shows a previously saved value", async () => { + const client = makeClient({ + getSsoConfig: vi.fn(() => + Promise.resolve( + makeConfig({ + idp: { + entity_id: "https://idp.acme.com/saml/metadata", + sso_url: "https://idp.acme.com/sso", + slo_url: "https://idp.acme.com/slo", + x509_certs: [ + "-----BEGIN CERTIFICATE-----\nAAA\n-----END CERTIFICATE-----", + ], + name_id_format: + "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress", + }, + }), + ), + ), + }); + renderView(client); + + const sloField = await screen.findByLabelText(/IdP SLO URL/i); + expect(sloField).toBeDisabled(); + // Disabled != hidden: an admin who previously saved a value should still see it. + expect(sloField).toHaveValue("https://idp.acme.com/slo"); + expect( + screen.getByText(/single logout is not yet supported/i), + ).toBeInTheDocument(); + // House style bans "coming soon" phrasing everywhere in this view. + expect(screen.queryByText(/coming soon/i)).not.toBeInTheDocument(); + + // The helper sentence is a sibling of the