feat: Self-Eval Gate — block config changes that regress a workspace's private per-repo suite (trust-layer 4/4)

.github/workflows/ci.yml

@@ -318,10 +318,15 @@ jobs:
           --error --metrics=off --sarif -o semgrep.sarif .
 
       # --- Dependency audit --- #
+      # PYSEC-2026-2132: command injection in click.edit(). Forge never imports
+      # click, so the vulnerable path is unreachable; click<8.2 is pinned by the
+      # semgrep dev/CI dependency, so click>=8.3.3 is not resolvable. Revisit and
+      # drop this ignore once semgrep lifts the click<8.2 cap (dependabot #1).
       - name: pip-audit
         run: >-
           uv export --frozen --format requirements-txt --no-emit-workspace
           | uv run pip-audit -r /dev/stdin --strict
+          --ignore-vuln PYSEC-2026-2132
 
       # --- Secret scan (full history) --- #
       # The gitleaks GitHub Action requires a paid license for ORGANIZATION

.github/workflows/security.yml

@@ -70,7 +70,12 @@ jobs:
         run: uv run bandit -c pyproject.toml -r packages apps --severity-level high -q
 
       # Dependency audit against the fully-resolved, frozen requirements set.
+      # PYSEC-2026-2132: command injection in click.edit(). Forge never imports
+      # click, so the vulnerable path is unreachable; click<8.2 is pinned by the
+      # semgrep dev/CI dependency, so click>=8.3.3 is not resolvable. Revisit and
+      # drop this ignore once semgrep lifts the click<8.2 cap (dependabot #1).
       - name: pip-audit
         run: >-
           uv export --frozen --format requirements-txt --no-emit-workspace
           | uv run pip-audit -r /dev/stdin --strict
+          --ignore-vuln PYSEC-2026-2132

apps/api/forge_api/routers/public_leaderboard.py

@@ -104,7 +104,7 @@ def list_public_benchmarks(service: ServiceDep, response: Response) -> list[Publ
             primary_metric=s.primary_metric,
             content_hash=s.content_hash,
         )
-        for s in service.list_suites(published_only=True)
+        for s in service.list_suites(published_only=True, public_only=True)
     ]
 
 

apps/api/forge_api/services/benchmark_service.py

@@ -170,13 +170,27 @@ def sync_suites_from_disk(self, *, published: bool = True) -> list[BenchmarkSuit
                 registered.append(self.register_suite(manifest, cases, published=published))
         return registered
 
-    def list_suites(self, *, published_only: bool = False) -> list[BenchmarkSuite]:
+    def list_suites(
+        self, *, published_only: bool = False, public_only: bool = False
+    ) -> list[BenchmarkSuite]:
         with self._session_factory() as session:
             stmt = select(BenchmarkSuite).order_by(BenchmarkSuite.slug, BenchmarkSuite.version)
             if published_only:
                 stmt = stmt.where(BenchmarkSuite.published.is_(True))
+            if public_only:
+                # Never expose private per-repo Self-Eval suites publicly: only
+                # global (workspace_id IS NULL), non-private suites are public.
+                stmt = stmt.where(
+                    BenchmarkSuite.workspace_id.is_(None),
+                    BenchmarkSuite.private.is_(False),
+                )
             return list(session.scalars(stmt).all())
 
+    @staticmethod
+    def is_public_suite(suite: BenchmarkSuite) -> bool:
+        """True iff the suite may be exposed via the public (unauthenticated) API."""
+        return suite.workspace_id is None and not suite.private
+
     def get_suite(self, slug: str, version: str) -> BenchmarkSuite:
         with self._session_factory() as session:
             suite = self._require_suite(session, slug, version)
@@ -407,7 +421,9 @@ def leaderboard(
     ) -> tuple[BenchmarkSuite, list[LeaderboardRow]]:
         with self._session_factory() as session:
             suite = self._require_suite(session, slug, version)
-            if public_only and not suite.published:
+            if public_only and (not suite.published or not self.is_public_suite(suite)):
+                # A private/workspace-scoped Self-Eval suite is invisible to the
+                # public API even if published — 404 as if it does not exist.
                 raise SuiteNotFoundError(f"{slug}@{version}")
             stmt = select(BenchmarkSubmission).where(
                 BenchmarkSubmission.benchmark_suite_id == suite.id,
@@ -437,6 +453,7 @@ def public_submission(
                 or submission.visibility != Visibility.public.value
                 or submission.status not in PUBLIC_RANKABLE_STATUSES
                 or not suite.published
+                or not self.is_public_suite(suite)
             ):
                 raise SubmissionNotFoundError(str(submission_id))
             return submission, suite

apps/api/tests/benchmark/test_public_leaderboard.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import pytest
-from conftest import SLUG, VERSION, faithful_submission
+from conftest import SLUG, VERSION, WS_ID, faithful_submission
 
 from forge_api.routers import public_leaderboard as public_module
 from forge_api.settings import Settings
@@ -130,3 +130,33 @@ def test_unpublished_suite_hidden_from_public(make_client, service) -> None:
     anon = make_client(None)
     assert anon.get("/public/benchmarks").json() == []
     assert anon.get(BOARD).status_code == 404
+
+
+@pytest.mark.parametrize(
+    "mutate",
+    [
+        pytest.param(lambda s: setattr(s, "private", True), id="private-flag"),
+        pytest.param(lambda s: setattr(s, "workspace_id", WS_ID), id="workspace-scoped"),
+    ],
+)
+def test_private_self_eval_suite_never_public(make_client, service, mutate) -> None:
+    """A private/workspace-scoped Self-Eval suite is invisible to /public/*.
+
+    Even when it is ``published`` (Self-Eval suites are published so the owning
+    workspace can rank against its own baseline), the public surface must treat
+    it as if it does not exist — no listing, no leaderboard.
+    """
+    sid = _published_submission(make_client)  # give the suite a rankable entry
+    with service._session_factory() as session:  # test-only reach-in
+        from forge_db.models.benchmark import BenchmarkSuite
+
+        suite = session.query(BenchmarkSuite).one()
+        assert suite.published is True  # precondition: only privacy hides it now
+        mutate(suite)
+        session.commit()
+
+    anon = make_client(None)
+    assert anon.get("/public/benchmarks").json() == []
+    assert anon.get(BOARD).status_code == 404
+    # The submission detail for that suite must 404 too — no back-door leak.
+    assert anon.get(f"{BOARD}/submissions/{sid}").status_code == 404

apps/worker/forge_worker/celery_app.py

@@ -42,6 +42,7 @@ def get_broker_url() -> str:
         "forge_worker.tasks.observability",
         "forge_worker.tasks.sso",
         "forge_worker.tasks.audit",
+        "forge_worker.tasks.self_eval_mint",
         "forge_worker.beat",
     ],
 )

apps/worker/forge_worker/tasks/self_eval_mint.py

@@ -0,0 +1,162 @@
+"""Self-Eval Gate (F41) minting worker task.
+
+As the org's own PRs merge, this task mints a hidden regression case from each
+(:func:`forge_eval.mint.mint_case_from_pr`) and appends it to the workspace's
+*private* benchmark suite dir, then **re-freezes** the manifest
+(:func:`forge_eval.benchmark.manifest.freeze`) so a fresh ``content_hash`` pins
+the grown case set. Unlike a published leaderboard suite, a per-repo private
+suite is a living accumulator: :func:`append_minted_cases` unfreezes, writes the
+minted case file, and re-freezes in one atomic step.
+
+The pure functions (:func:`append_minted_cases`, :func:`mint_and_store`) carry
+the logic and are exercised offline in tests; the ``@celery_app.task`` wrapper is
+the thin prod seam. Minted hidden test ids live only on disk in the suite dir and
+never enter a model prompt.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+from forge_eval.benchmark.manifest import freeze, load_manifest
+from forge_eval.golden import GoldenCase, load_golden_set
+from forge_eval.mint import PullRequestSource, TestRunner, mint_case_from_pr
+from forge_worker.celery_app import celery_app
+
+__all__ = [
+    "MINTED_CASE_FILE",
+    "append_minted_cases",
+    "mint_and_store",
+    "self_eval_mint_task",
+]
+
+#: Relative path (within a suite version dir) the minted cases accumulate into.
+MINTED_CASE_FILE = "cases/self_eval.json"
+
+
+def _case_to_dict(case: GoldenCase) -> dict[str, Any]:
+    return asdict(case)
+
+
+def _load_case_files(version_dir: Path, case_files: list[str]) -> list[GoldenCase]:
+    cases: list[GoldenCase] = []
+    seen: set[str] = set()
+    for rel in case_files:
+        for case in load_golden_set(version_dir / rel):
+            if case.id in seen:
+                raise ValueError(f"duplicate benchmark case id across files: {case.id!r}")
+            seen.add(case.id)
+            cases.append(case)
+    return cases
+
+
+def append_minted_cases(
+    version_dir: str | Path,
+    new_cases: list[GoldenCase],
+    *,
+    minted_case_file: str = MINTED_CASE_FILE,
+) -> int:
+    """Append ``new_cases`` to a private suite's minted case file + re-freeze.
+
+    Idempotent on case id: a minted case whose id already exists is updated in
+    place, not duplicated. Writes the merged manifest back to
+    ``version_dir/manifest.yaml`` with a freshly recomputed ``content_hash``.
+    Returns the number of *newly added* cases.
+    """
+    resolved = Path(version_dir)
+    manifest, _existing = load_manifest(resolved)
+
+    minted_path = resolved / minted_case_file
+    prior = load_golden_set(minted_path) if minted_path.is_file() else []
+    by_id: dict[str, GoldenCase] = {c.id: c for c in prior}
+    added = 0
+    for case in new_cases:
+        if case.id not in by_id:
+            added += 1
+        by_id[case.id] = case
+    minted = sorted(by_id.values(), key=lambda c: c.id)
+
+    minted_path.parent.mkdir(parents=True, exist_ok=True)
+    minted_path.write_text(
+        json.dumps([_case_to_dict(c) for c in minted], indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+    case_files = list(manifest.case_files)
+    if minted_case_file not in case_files:
+        case_files.append(minted_case_file)
+
+    all_cases = _load_case_files(resolved, case_files)
+    unfrozen = manifest.model_copy(
+        update={"frozen": False, "content_hash": None, "case_files": case_files}
+    )
+    reforged = freeze(unfrozen, all_cases)
+
+    import yaml  # lazy: mirrors manifest.py's optional PyYAML dependency
+
+    (resolved / "manifest.yaml").write_text(
+        yaml.safe_dump(reforged.model_dump(mode="json"), sort_keys=False),
+        encoding="utf-8",
+    )
+    return added
+
+
+def mint_and_store(
+    prs: list[Any],
+    repo: str,
+    version_dir: str | Path,
+    *,
+    source: PullRequestSource,
+    runner: TestRunner,
+    sandbox_image: str | None = None,
+    setup_commands: list[str] | None = None,
+    max_pass_to_pass: int = 10,
+) -> list[str]:
+    """Mint a case per merged PR and append the non-empty ones to the suite.
+
+    Returns the ids of the cases that were minted (PRs with no fail-to-pass
+    signal are silently skipped — not faked). PR data and test outcomes are
+    injected, so this runs fully offline in tests.
+    """
+    minted: list[GoldenCase] = []
+    for pr in prs:
+        case = mint_case_from_pr(
+            pr,
+            repo,
+            source=source,
+            runner=runner,
+            sandbox_image=sandbox_image,
+            setup_commands=setup_commands or [],
+            max_pass_to_pass=max_pass_to_pass,
+        )
+        if case is not None:
+            minted.append(case)
+    if minted:
+        append_minted_cases(version_dir, minted)
+    return [c.id for c in minted]
+
+
+@celery_app.task(name="forge.self_eval.mint", queue="self_eval")
+def self_eval_mint_task(repo: str, pr_number: int, version_dir: str) -> dict[str, Any]:
+    """Prod seam: mint a case for one merged PR into a workspace private suite.
+
+    Builds a GitHub-backed :class:`PullRequestSource` + worktree
+    :class:`TestRunner` and delegates to :func:`mint_and_store`. Import of the
+    integration SDK is lazy so the pure minting logic (and its tests) never pull
+    ``httpx``/``forge_integrations`` into scope.
+    """
+    import os
+
+    from forge_contracts import PullRequest
+    from forge_eval.mint import GitHubPullRequestSource, GitWorktreeTestRunner
+    from forge_integrations.github import GitHubClient
+
+    client = GitHubClient(token=os.environ.get("GITHUB_TOKEN"))
+    source = GitHubPullRequestSource(client=client)
+    runner = GitWorktreeTestRunner(repo_path=version_dir)
+    pr = PullRequest(repo=repo, number=pr_number, head_sha=client.pr_head_commit(repo, pr_number))
+    minted = mint_and_store([pr], repo, version_dir, source=source, runner=runner)
+    return {"repo": repo, "pr": pr_number, "minted": minted}

apps/worker/pyproject.toml

@@ -26,6 +26,11 @@ dependencies = [
     "celery>=5.3",
     "redis>=5.0",
     "forge-approval",
+    # F41 — Self-Eval Gate: mint hidden regression cases into private suites and
+    # re-freeze their manifests (forge_eval.mint + benchmark.manifest.freeze);
+    # forge-integrations supplies the GitHub PR-data source in the prod seam.
+    "forge-eval",
+    "forge-integrations",
 ]
 
 [build-system]