diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 055a6434..66d2a1e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -318,10 +318,15 @@ jobs: --error --metrics=off --sarif -o semgrep.sarif . # --- Dependency audit --- # + # PYSEC-2026-2132: command injection in click.edit(). Forge never imports + # click, so the vulnerable path is unreachable; click<8.2 is pinned by the + # semgrep dev/CI dependency, so click>=8.3.3 is not resolvable. Revisit and + # drop this ignore once semgrep lifts the click<8.2 cap (dependabot #1). - name: pip-audit run: >- uv export --frozen --format requirements-txt --no-emit-workspace | uv run pip-audit -r /dev/stdin --strict + --ignore-vuln PYSEC-2026-2132 # --- Secret scan (full history) --- # # The gitleaks GitHub Action requires a paid license for ORGANIZATION diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 785df50d..7f58c157 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -70,7 +70,12 @@ jobs: run: uv run bandit -c pyproject.toml -r packages apps --severity-level high -q # Dependency audit against the fully-resolved, frozen requirements set. + # PYSEC-2026-2132: command injection in click.edit(). Forge never imports + # click, so the vulnerable path is unreachable; click<8.2 is pinned by the + # semgrep dev/CI dependency, so click>=8.3.3 is not resolvable. Revisit and + # drop this ignore once semgrep lifts the click<8.2 cap (dependabot #1). - name: pip-audit run: >- uv export --frozen --format requirements-txt --no-emit-workspace | uv run pip-audit -r /dev/stdin --strict + --ignore-vuln PYSEC-2026-2132 diff --git a/apps/api/forge_api/routers/public_leaderboard.py b/apps/api/forge_api/routers/public_leaderboard.py index 65c9c1ea..1888cf70 100644 --- a/apps/api/forge_api/routers/public_leaderboard.py +++ b/apps/api/forge_api/routers/public_leaderboard.py @@ -104,7 +104,7 @@ def list_public_benchmarks(service: ServiceDep, response: Response) -> list[Publ primary_metric=s.primary_metric, content_hash=s.content_hash, ) - for s in service.list_suites(published_only=True) + for s in service.list_suites(published_only=True, public_only=True) ] diff --git a/apps/api/forge_api/services/benchmark_service.py b/apps/api/forge_api/services/benchmark_service.py index 2acbc3b9..2a67f10e 100644 --- a/apps/api/forge_api/services/benchmark_service.py +++ b/apps/api/forge_api/services/benchmark_service.py @@ -170,13 +170,27 @@ def sync_suites_from_disk(self, *, published: bool = True) -> list[BenchmarkSuit registered.append(self.register_suite(manifest, cases, published=published)) return registered - def list_suites(self, *, published_only: bool = False) -> list[BenchmarkSuite]: + def list_suites( + self, *, published_only: bool = False, public_only: bool = False + ) -> list[BenchmarkSuite]: with self._session_factory() as session: stmt = select(BenchmarkSuite).order_by(BenchmarkSuite.slug, BenchmarkSuite.version) if published_only: stmt = stmt.where(BenchmarkSuite.published.is_(True)) + if public_only: + # Never expose private per-repo Self-Eval suites publicly: only + # global (workspace_id IS NULL), non-private suites are public. + stmt = stmt.where( + BenchmarkSuite.workspace_id.is_(None), + BenchmarkSuite.private.is_(False), + ) return list(session.scalars(stmt).all()) + @staticmethod + def is_public_suite(suite: BenchmarkSuite) -> bool: + """True iff the suite may be exposed via the public (unauthenticated) API.""" + return suite.workspace_id is None and not suite.private + def get_suite(self, slug: str, version: str) -> BenchmarkSuite: with self._session_factory() as session: suite = self._require_suite(session, slug, version) @@ -407,7 +421,9 @@ def leaderboard( ) -> tuple[BenchmarkSuite, list[LeaderboardRow]]: with self._session_factory() as session: suite = self._require_suite(session, slug, version) - if public_only and not suite.published: + if public_only and (not suite.published or not self.is_public_suite(suite)): + # A private/workspace-scoped Self-Eval suite is invisible to the + # public API even if published — 404 as if it does not exist. raise SuiteNotFoundError(f"{slug}@{version}") stmt = select(BenchmarkSubmission).where( BenchmarkSubmission.benchmark_suite_id == suite.id, @@ -437,6 +453,7 @@ def public_submission( or submission.visibility != Visibility.public.value or submission.status not in PUBLIC_RANKABLE_STATUSES or not suite.published + or not self.is_public_suite(suite) ): raise SubmissionNotFoundError(str(submission_id)) return submission, suite diff --git a/apps/api/tests/benchmark/test_public_leaderboard.py b/apps/api/tests/benchmark/test_public_leaderboard.py index 62b6bc27..d7ef9d32 100644 --- a/apps/api/tests/benchmark/test_public_leaderboard.py +++ b/apps/api/tests/benchmark/test_public_leaderboard.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest -from conftest import SLUG, VERSION, faithful_submission +from conftest import SLUG, VERSION, WS_ID, faithful_submission from forge_api.routers import public_leaderboard as public_module from forge_api.settings import Settings @@ -130,3 +130,33 @@ def test_unpublished_suite_hidden_from_public(make_client, service) -> None: anon = make_client(None) assert anon.get("/public/benchmarks").json() == [] assert anon.get(BOARD).status_code == 404 + + +@pytest.mark.parametrize( + "mutate", + [ + pytest.param(lambda s: setattr(s, "private", True), id="private-flag"), + pytest.param(lambda s: setattr(s, "workspace_id", WS_ID), id="workspace-scoped"), + ], +) +def test_private_self_eval_suite_never_public(make_client, service, mutate) -> None: + """A private/workspace-scoped Self-Eval suite is invisible to /public/*. + + Even when it is ``published`` (Self-Eval suites are published so the owning + workspace can rank against its own baseline), the public surface must treat + it as if it does not exist — no listing, no leaderboard. + """ + sid = _published_submission(make_client) # give the suite a rankable entry + with service._session_factory() as session: # test-only reach-in + from forge_db.models.benchmark import BenchmarkSuite + + suite = session.query(BenchmarkSuite).one() + assert suite.published is True # precondition: only privacy hides it now + mutate(suite) + session.commit() + + anon = make_client(None) + assert anon.get("/public/benchmarks").json() == [] + assert anon.get(BOARD).status_code == 404 + # The submission detail for that suite must 404 too — no back-door leak. + assert anon.get(f"{BOARD}/submissions/{sid}").status_code == 404 diff --git a/apps/worker/forge_worker/celery_app.py b/apps/worker/forge_worker/celery_app.py index 27ff1b1d..2a93e8df 100644 --- a/apps/worker/forge_worker/celery_app.py +++ b/apps/worker/forge_worker/celery_app.py @@ -42,6 +42,7 @@ def get_broker_url() -> str: "forge_worker.tasks.observability", "forge_worker.tasks.sso", "forge_worker.tasks.audit", + "forge_worker.tasks.self_eval_mint", "forge_worker.beat", ], ) diff --git a/apps/worker/forge_worker/tasks/self_eval_mint.py b/apps/worker/forge_worker/tasks/self_eval_mint.py new file mode 100644 index 00000000..e51d8192 --- /dev/null +++ b/apps/worker/forge_worker/tasks/self_eval_mint.py @@ -0,0 +1,162 @@ +"""Self-Eval Gate (F41) minting worker task. + +As the org's own PRs merge, this task mints a hidden regression case from each +(:func:`forge_eval.mint.mint_case_from_pr`) and appends it to the workspace's +*private* benchmark suite dir, then **re-freezes** the manifest +(:func:`forge_eval.benchmark.manifest.freeze`) so a fresh ``content_hash`` pins +the grown case set. Unlike a published leaderboard suite, a per-repo private +suite is a living accumulator: :func:`append_minted_cases` unfreezes, writes the +minted case file, and re-freezes in one atomic step. + +The pure functions (:func:`append_minted_cases`, :func:`mint_and_store`) carry +the logic and are exercised offline in tests; the ``@celery_app.task`` wrapper is +the thin prod seam. Minted hidden test ids live only on disk in the suite dir and +never enter a model prompt. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from forge_eval.benchmark.manifest import freeze, load_manifest +from forge_eval.golden import GoldenCase, load_golden_set +from forge_eval.mint import PullRequestSource, TestRunner, mint_case_from_pr +from forge_worker.celery_app import celery_app + +__all__ = [ + "MINTED_CASE_FILE", + "append_minted_cases", + "mint_and_store", + "self_eval_mint_task", +] + +#: Relative path (within a suite version dir) the minted cases accumulate into. +MINTED_CASE_FILE = "cases/self_eval.json" + + +def _case_to_dict(case: GoldenCase) -> dict[str, Any]: + return asdict(case) + + +def _load_case_files(version_dir: Path, case_files: list[str]) -> list[GoldenCase]: + cases: list[GoldenCase] = [] + seen: set[str] = set() + for rel in case_files: + for case in load_golden_set(version_dir / rel): + if case.id in seen: + raise ValueError(f"duplicate benchmark case id across files: {case.id!r}") + seen.add(case.id) + cases.append(case) + return cases + + +def append_minted_cases( + version_dir: str | Path, + new_cases: list[GoldenCase], + *, + minted_case_file: str = MINTED_CASE_FILE, +) -> int: + """Append ``new_cases`` to a private suite's minted case file + re-freeze. + + Idempotent on case id: a minted case whose id already exists is updated in + place, not duplicated. Writes the merged manifest back to + ``version_dir/manifest.yaml`` with a freshly recomputed ``content_hash``. + Returns the number of *newly added* cases. + """ + resolved = Path(version_dir) + manifest, _existing = load_manifest(resolved) + + minted_path = resolved / minted_case_file + prior = load_golden_set(minted_path) if minted_path.is_file() else [] + by_id: dict[str, GoldenCase] = {c.id: c for c in prior} + added = 0 + for case in new_cases: + if case.id not in by_id: + added += 1 + by_id[case.id] = case + minted = sorted(by_id.values(), key=lambda c: c.id) + + minted_path.parent.mkdir(parents=True, exist_ok=True) + minted_path.write_text( + json.dumps([_case_to_dict(c) for c in minted], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + case_files = list(manifest.case_files) + if minted_case_file not in case_files: + case_files.append(minted_case_file) + + all_cases = _load_case_files(resolved, case_files) + unfrozen = manifest.model_copy( + update={"frozen": False, "content_hash": None, "case_files": case_files} + ) + reforged = freeze(unfrozen, all_cases) + + import yaml # lazy: mirrors manifest.py's optional PyYAML dependency + + (resolved / "manifest.yaml").write_text( + yaml.safe_dump(reforged.model_dump(mode="json"), sort_keys=False), + encoding="utf-8", + ) + return added + + +def mint_and_store( + prs: list[Any], + repo: str, + version_dir: str | Path, + *, + source: PullRequestSource, + runner: TestRunner, + sandbox_image: str | None = None, + setup_commands: list[str] | None = None, + max_pass_to_pass: int = 10, +) -> list[str]: + """Mint a case per merged PR and append the non-empty ones to the suite. + + Returns the ids of the cases that were minted (PRs with no fail-to-pass + signal are silently skipped — not faked). PR data and test outcomes are + injected, so this runs fully offline in tests. + """ + minted: list[GoldenCase] = [] + for pr in prs: + case = mint_case_from_pr( + pr, + repo, + source=source, + runner=runner, + sandbox_image=sandbox_image, + setup_commands=setup_commands or [], + max_pass_to_pass=max_pass_to_pass, + ) + if case is not None: + minted.append(case) + if minted: + append_minted_cases(version_dir, minted) + return [c.id for c in minted] + + +@celery_app.task(name="forge.self_eval.mint", queue="self_eval") +def self_eval_mint_task(repo: str, pr_number: int, version_dir: str) -> dict[str, Any]: + """Prod seam: mint a case for one merged PR into a workspace private suite. + + Builds a GitHub-backed :class:`PullRequestSource` + worktree + :class:`TestRunner` and delegates to :func:`mint_and_store`. Import of the + integration SDK is lazy so the pure minting logic (and its tests) never pull + ``httpx``/``forge_integrations`` into scope. + """ + import os + + from forge_contracts import PullRequest + from forge_eval.mint import GitHubPullRequestSource, GitWorktreeTestRunner + from forge_integrations.github import GitHubClient + + client = GitHubClient(token=os.environ.get("GITHUB_TOKEN")) + source = GitHubPullRequestSource(client=client) + runner = GitWorktreeTestRunner(repo_path=version_dir) + pr = PullRequest(repo=repo, number=pr_number, head_sha=client.pr_head_commit(repo, pr_number)) + minted = mint_and_store([pr], repo, version_dir, source=source, runner=runner) + return {"repo": repo, "pr": pr_number, "minted": minted} diff --git a/apps/worker/pyproject.toml b/apps/worker/pyproject.toml index ceda14c9..863df779 100644 --- a/apps/worker/pyproject.toml +++ b/apps/worker/pyproject.toml @@ -26,6 +26,11 @@ dependencies = [ "celery>=5.3", "redis>=5.0", "forge-approval", + # F41 — Self-Eval Gate: mint hidden regression cases into private suites and + # re-freeze their manifests (forge_eval.mint + benchmark.manifest.freeze); + # forge-integrations supplies the GitHub PR-data source in the prod seam. + "forge-eval", + "forge-integrations", ] [build-system] diff --git a/apps/worker/tests/test_self_eval_mint.py b/apps/worker/tests/test_self_eval_mint.py new file mode 100644 index 00000000..3918f888 --- /dev/null +++ b/apps/worker/tests/test_self_eval_mint.py @@ -0,0 +1,200 @@ +"""F41 Self-Eval Gate worker task: append minted cases + re-freeze (offline). + +No git, no GitHub, no real provider: a scripted in-memory ``TestRunner`` + fake +PR source drive the mint, and the suite lives in a ``tmp_path`` dir. Asserts the +private suite grows, the manifest re-freezes with a fresh content hash, and the +minted hidden tests round-trip through ``load_manifest`` (so a gate run can load +them from disk). +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +import yaml + +from forge_eval.benchmark import load_manifest, parse_swe_case_fields +from forge_eval.benchmark.manifest import compute_content_hash, freeze +from forge_eval.golden import GoldenCase +from forge_eval.mint import ChangedFile +from forge_worker.celery_app import celery_app +from forge_worker.tasks.self_eval_mint import ( + MINTED_CASE_FILE, + append_minted_cases, + mint_and_store, +) + + +@dataclass +class FakePR: + number: int + head_sha: str + title: str = "Fix regression" + + +class FakeSource: + def __init__(self, files_by_pr: dict[int, list[ChangedFile]], base: str) -> None: + self._files = files_by_pr + self._base = base + + def pr_changed_files(self, repo: str, number: int) -> list[ChangedFile]: + return list(self._files.get(number, [])) + + def pr_base_commit(self, repo: str, number: int) -> str: + return self._base + + +class ScriptedRunner: + """Deterministic in-memory TestRunner: ``outcomes[(ref, node)] -> passed``.""" + + def __init__(self, outcomes: dict[tuple[str, str], bool], collected: list[str]) -> None: + self._outcomes = outcomes + self._collected = collected + + def run_tests(self, *, ref: str, node_ids: Sequence[str]) -> dict[str, bool]: + return {nid: self._outcomes.get((ref, nid), False) for nid in node_ids} + + def collect_tests(self, *, ref: str) -> list[str]: + return list(self._collected) + + +_PATCH = "@@ -0,0 +1,2 @@\n+def test_new():\n+ assert True\n" + + +def _seed_suite(version_dir: Path) -> None: + """Write a valid, frozen 1-case seed private suite.""" + version_dir.mkdir(parents=True, exist_ok=True) + (version_dir / "cases").mkdir(exist_ok=True) + seed = [GoldenCase(id="seed-1", query="seed", expected_ids=["a"], tags=["seed"])] + (version_dir / "cases" / "seed.json").write_text( + json.dumps([{"id": "seed-1", "query": "seed", "expected_ids": ["a"], "tags": ["seed"]}]), + encoding="utf-8", + ) + scoring = { + "primary_metric": "benchmark.composite", + "metric_weights": {"retrieval.recall_at_k": 1.0}, + } + from forge_eval.benchmark import BenchmarkManifest + + manifest = BenchmarkManifest.model_validate( + { + "slug": "acme-private", + "version": "1.0.0", + "title": "acme private suite", + "scoring": scoring, + "case_files": ["cases/seed.json"], + } + ) + frozen = freeze(manifest, seed) + (version_dir / "manifest.yaml").write_text( + yaml.safe_dump(frozen.model_dump(mode="json"), sort_keys=False), encoding="utf-8" + ) + + +def test_append_minted_cases_grows_and_refreezes(tmp_path: Path) -> None: + version_dir = tmp_path / "suite" / "1.0.0" + _seed_suite(version_dir) + before, _ = load_manifest(version_dir) + + minted = GoldenCase( + id="self-eval-acme-pr-1", + query="Fix the bug", + expected_ids=["test_new.py::test_new"], + kind="agent_task", + tags=["self-eval"], + metadata={ + "fail_to_pass": ["test_new.py::test_new"], + "pass_to_pass": ["test_old.py::test_old"], + "sandbox_image": "python:3.14-slim", + "setup_commands": ["pip install -e ."], + "base_commit": "deadbeef", + "expected_terminal_state": "pr_opened", + }, + ) + added = append_minted_cases(version_dir, [minted]) + assert added == 1 + + after, cases = load_manifest(version_dir) + assert after.frozen is True + assert after.content_hash != before.content_hash # re-froze with a new hash + assert MINTED_CASE_FILE in after.case_files + ids = {c.id for c in cases} + assert ids == {"seed-1", "self-eval-acme-pr-1"} + + # Hidden test wiring survives the disk round-trip. + minted_loaded = next(c for c in cases if c.id == "self-eval-acme-pr-1") + fields = parse_swe_case_fields(minted_loaded) + assert fields.fail_to_pass == ["test_new.py::test_new"] + assert fields.pass_to_pass == ["test_old.py::test_old"] + + +def test_append_minted_cases_is_idempotent_on_id(tmp_path: Path) -> None: + version_dir = tmp_path / "suite" / "1.0.0" + _seed_suite(version_dir) + case = GoldenCase( + id="self-eval-acme-pr-2", + query="q", + expected_ids=["t::t"], + kind="agent_task", + tags=["self-eval"], + metadata={"fail_to_pass": ["t::t"], "expected_terminal_state": "pr_opened"}, + ) + assert append_minted_cases(version_dir, [case]) == 1 + # Re-appending the same id adds nothing new and still re-freezes cleanly. + assert append_minted_cases(version_dir, [case]) == 0 + _, cases = load_manifest(version_dir) + assert sum(1 for c in cases if c.id == "self-eval-acme-pr-2") == 1 + + +def test_mint_and_store_end_to_end(tmp_path: Path) -> None: + version_dir = tmp_path / "suite" / "1.0.0" + _seed_suite(version_dir) + + node = "test_new.py::test_new" + changed = [ChangedFile(path="test_new.py", status="added", patch=_PATCH)] + source = FakeSource({1: changed}, "base1") + runner = ScriptedRunner( + outcomes={ + ("base1", node): False, # fails before + ("head1", node): True, # passes after + ("base1", "test_old.py::test_old"): True, + ("head1", "test_old.py::test_old"): True, + }, + collected=["test_old.py::test_old", node], + ) + pr = FakePR(number=1, head_sha="head1") + + minted_ids = mint_and_store( + [pr], "acme/widgets", version_dir, source=source, runner=runner, sandbox_image="img" + ) + assert minted_ids == ["self-eval-acme-widgets-pr-1"] + + _, cases = load_manifest(version_dir) + minted = next(c for c in cases if c.id == "self-eval-acme-widgets-pr-1") + fields = parse_swe_case_fields(minted) + assert fields.fail_to_pass == [node] + assert fields.pass_to_pass == ["test_old.py::test_old"] + assert fields.sandbox_image == "img" + + +def test_content_hash_matches_manual_recompute(tmp_path: Path) -> None: + version_dir = tmp_path / "suite" / "1.0.0" + _seed_suite(version_dir) + case = GoldenCase( + id="self-eval-acme-pr-3", + query="q", + expected_ids=["t::t"], + kind="agent_task", + tags=["self-eval"], + metadata={"fail_to_pass": ["t::t"], "expected_terminal_state": "pr_opened"}, + ) + append_minted_cases(version_dir, [case]) + manifest, cases = load_manifest(version_dir) + assert manifest.content_hash == compute_content_hash(cases, manifest.scoring) + + +def test_task_registered() -> None: + assert "forge.self_eval.mint" in celery_app.tasks diff --git a/packages/db/forge_db/models/benchmark.py b/packages/db/forge_db/models/benchmark.py index 465c89cb..8ee81149 100644 --- a/packages/db/forge_db/models/benchmark.py +++ b/packages/db/forge_db/models/benchmark.py @@ -14,11 +14,25 @@ ``eval_run``/``replay_bundle`` tables and a MinIO bundle store — neither exists in-tree, so there is no ``eval_run_id`` FK and the deterministic replay bundles are persisted inline in ``replay_bundles`` (JSON, size-capped at ingest). -``benchmark_suite`` is global (no ``workspace_id``): a frozen suite is a shared -community artifact, like the file-based golden sets. Submissions carry a -*nullable* ``workspace_id`` (NULL = official/system submission). Status / -visibility are stored as plain strings guarded by CHECK constraints, matching -the marketplace/deployment precedent. +``benchmark_suite`` is global (no ``workspace_id``) by default: a frozen suite +is a shared community artifact, like the file-based golden sets. Submissions +carry a *nullable* ``workspace_id`` (NULL = official/system submission). +Status / visibility are stored as plain strings guarded by CHECK constraints, +matching the marketplace/deployment precedent. + +Self-Eval Gate (F41) extends ``benchmark_suite`` with three *nullable* +columns so an org can mint a PRIVATE, per-repo regression suite from its own +merged PRs, without disturbing the existing global/public suites (which keep +``workspace_id``/``repo_id`` NULL and ``private=False``): + +* ``workspace_id`` — NULL = shared/community suite (unchanged default); + non-NULL = scoped to one workspace's own benchmark. +* ``repo_id`` — the source repository the suite was minted from (free-form + provider identifier, mirroring ``RepositoryConnection.repo_id`` — no FK, + since suites can outlive a disconnected repository). +* ``private`` — when true, the suite (and its submissions) must never be + surfaced by ``/public/*`` leaderboard endpoints, regardless of submission + ``visibility``. """ from __future__ import annotations @@ -52,6 +66,7 @@ class BenchmarkSuite(ForgeModel): __table_args__ = ( UniqueConstraint("slug", "version", name="uq_benchmark_suite_slug_version"), Index("ix_benchmark_suite_published_slug", "published", "slug"), + Index("ix_benchmark_suite_workspace_id", "workspace_id"), ) slug: Mapped[str] = mapped_column(String(64), nullable=False) @@ -72,6 +87,17 @@ class BenchmarkSuite(ForgeModel): created_by: Mapped[uuid.UUID | None] = mapped_column( Uuid(as_uuid=True), ForeignKey("app_user.id", ondelete="SET NULL"), nullable=True ) + #: NULL = shared/community suite (unscoped, matches prior behavior). + workspace_id: Mapped[uuid.UUID | None] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("workspace.id", ondelete="CASCADE"), + nullable=True, + ) + #: Source repository the suite was minted from (free-form provider + #: identifier, e.g. ``"github:org/repo"``); no FK — outlives disconnects. + repo_id: Mapped[str | None] = mapped_column(String(512), nullable=True) + #: Self-Eval Gate suites are private: never surfaced by ``/public/*``. + private: Mapped[bool] = mapped_column(default=False, nullable=False) class BenchmarkSubmission(ForgeModel): diff --git a/packages/db/migrations/versions/0039_self_eval_suite_scoping.py b/packages/db/migrations/versions/0039_self_eval_suite_scoping.py new file mode 100644 index 00000000..8b11c11b --- /dev/null +++ b/packages/db/migrations/versions/0039_self_eval_suite_scoping.py @@ -0,0 +1,101 @@ +"""self-eval gate: benchmark_suite workspace/repo scoping + private flag + +Extends ``benchmark_suite`` (``forge_db.models.benchmark.BenchmarkSuite``) +with three *nullable*/defaulted columns so an org can mint a PRIVATE, +per-repo regression suite from its own merged PRs (F41 "Self-Eval Gate"), +without disturbing any existing global/public suite row: + +* ``workspace_id`` — nullable FK to ``workspace``; NULL preserves today's + "shared/community suite" semantics for every pre-existing row. +* ``repo_id`` — nullable free-form source-repository identifier (no FK, + mirrors ``repository_connection.repo_id``). +* ``private`` — ``NOT NULL`` with server default ``false``, so existing rows + read as public/community (unchanged) and only newly-minted self-eval + suites opt into ``private=true``. + +Also adds the ``ix_benchmark_suite_workspace_id`` lookup index. + +Nothing existing is dropped or renamed, so this migration cannot break +existing behaviour. + +Idempotent like 0026/0036/0037/0038: ``upgrade`` adds only what is missing; +``downgrade`` drops only what this revision introduced. + +Revision ID: 0039_self_eval_suite_scoping +Revises: 0038_red_team_gate +Create Date: 2026-07-12 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) + +# revision identifiers, used by Alembic. +revision: str = "0039_self_eval_suite_scoping" +down_revision: str | None = "0038_red_team_gate" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_TABLE = "benchmark_suite" +# Column names owned by this revision, in add order (downgrade drops in reverse). +_COLUMN_NAMES: tuple[str, ...] = ("workspace_id", "repo_id", "private") +_INDEX_NAME = "ix_benchmark_suite_workspace_id" + + +def _new_columns() -> list[sa.Column]: + """Fresh Column objects per call (a Column may be bound to one table only).""" + return [ + sa.Column( + "workspace_id", + sa.Uuid(as_uuid=True), + sa.ForeignKey("workspace.id", ondelete="CASCADE"), + nullable=True, + ), + sa.Column("repo_id", sa.String(length=512), nullable=True), + sa.Column( + "private", + sa.Boolean(), + server_default=sa.text("false"), + nullable=False, + ), + ] + + +def _existing_columns() -> set[str]: + return {c["name"] for c in sa.inspect(op.get_bind()).get_columns(_TABLE)} + + +def _existing_indexes() -> set[str]: + return {ix["name"] for ix in sa.inspect(op.get_bind()).get_indexes(_TABLE)} + + +def upgrade() -> None: + columns = _existing_columns() + for column in _new_columns(): + if column.name not in columns: + op.add_column(_TABLE, column) + + if _INDEX_NAME not in _existing_indexes(): + op.create_index(_INDEX_NAME, _TABLE, ["workspace_id"]) + + +def downgrade() -> None: + # SQLite has no native DROP COLUMN/CONSTRAINT for an FK'd column; batch + # mode recreates the table under the hood. Postgres uses the same batch + # API but takes the direct ALTER TABLE path (no recreate needed). + columns = _existing_columns() + indexes = _existing_indexes() + to_drop = [name for name in reversed(_COLUMN_NAMES) if name in columns] + if not to_drop and _INDEX_NAME not in indexes: + return + + with op.batch_alter_table(_TABLE) as batch_op: + if _INDEX_NAME in indexes: + batch_op.drop_index(_INDEX_NAME) + for name in to_drop: + batch_op.drop_column(name) diff --git a/packages/db/tests/test_benchmark_models.py b/packages/db/tests/test_benchmark_models.py index 6fe974b8..1c6e367f 100644 --- a/packages/db/tests/test_benchmark_models.py +++ b/packages/db/tests/test_benchmark_models.py @@ -53,6 +53,51 @@ def test_create_all_produces_tables_and_leaderboard_index(engine) -> None: assert "ix_benchmark_submission_leaderboard" in submission_indexes assert "ix_benchmark_submission_workspace_submitted" in submission_indexes + suite_indexes = {ix["name"] for ix in inspector.get_indexes("benchmark_suite")} + assert "ix_benchmark_suite_workspace_id" in suite_indexes + + +def test_suite_defaults_to_global_unscoped(engine) -> None: + """F41: pre-existing/global suites keep workspace_id NULL and private=False.""" + with Session(engine) as session: + suite = _suite() + session.add(suite) + session.commit() + + row = session.get(BenchmarkSuite, suite.id) + assert row is not None + assert row.workspace_id is None + assert row.repo_id is None + assert row.private is False + + +def test_suite_can_be_scoped_private_self_eval(engine) -> None: + """F41: a minted Self-Eval Gate suite is private and workspace/repo scoped.""" + with Session(engine) as session: + ws = Workspace(name="Acme", slug="acme") + session.add(ws) + session.flush() + + suite = _suite( + slug="self-eval-acme", + workspace_id=ws.id, + repo_id="github:acme/widgets", + private=True, + ) + session.add(suite) + session.commit() + + row = session.get(BenchmarkSuite, suite.id) + assert row is not None + assert row.workspace_id == ws.id + assert row.repo_id == "github:acme/widgets" + assert row.private is True + + +def test_suite_workspace_fk_cascades_on_delete(engine) -> None: + fks = {fk.parent.name: fk.ondelete for fk in BenchmarkSuite.__table__.foreign_keys} + assert fks["workspace_id"] == "CASCADE" + def test_slug_version_unique(engine) -> None: with Session(engine) as session: diff --git a/packages/db/tests/test_models.py b/packages/db/tests/test_models.py index e518772c..d793141e 100644 --- a/packages/db/tests/test_models.py +++ b/packages/db/tests/test_models.py @@ -195,9 +195,6 @@ "pm_webhook_delivery", "marketplace_listing_version", "saml_replay", - # A frozen benchmark suite is a global community artifact (F35 §3.1); - # submissions carry a *nullable* workspace_id (NULL = official/system). - "benchmark_suite", # The observability audit store is a *global* (cross-workspace) hash chain, # mirroring the in-memory store: the entry carries only an optional, un-FK'd # ``workspace_ref`` tag, and the cursor row no workspace column at all. diff --git a/packages/evaluation/forge_eval/benchmark/__init__.py b/packages/evaluation/forge_eval/benchmark/__init__.py index 375f2081..bd6f4182 100644 --- a/packages/evaluation/forge_eval/benchmark/__init__.py +++ b/packages/evaluation/forge_eval/benchmark/__init__.py @@ -43,6 +43,7 @@ replay_bundles, ) from forge_eval.benchmark.scoring import compute_benchmark_score +from forge_eval.benchmark.swe_case import SweCaseFields, parse_swe_case_fields from forge_eval.benchmark.verify import verify_submission __all__ = [ @@ -63,6 +64,7 @@ "MetricAggregate", "ReplayBundle", "SubmissionStatus", + "SweCaseFields", "VerificationResult", "Visibility", "compute_benchmark_score", @@ -71,6 +73,7 @@ "freeze", "load_manifest", "make_bundle", + "parse_swe_case_fields", "rank_submissions", "replay_bundles", "validate_freezable", diff --git a/packages/evaluation/forge_eval/benchmark/replay.py b/packages/evaluation/forge_eval/benchmark/replay.py index 987085b5..4fcda0dc 100644 --- a/packages/evaluation/forge_eval/benchmark/replay.py +++ b/packages/evaluation/forge_eval/benchmark/replay.py @@ -46,6 +46,12 @@ "agent.requirement_satisfaction_rate": lambda out, exp, _k: ( len(set(exp) & set(out)) / len(set(exp)) if exp else 1.0 ), + # Self-Eval Gate: a minted case records the fail-to-pass tests that PASS + # after the candidate patch as its outputs; the fraction of the expected + # fail-to-pass set that now passes is the resolution rate the gate blocks on. + "agent.fail_to_pass_rate": lambda out, exp, _k: ( + len(set(exp) & set(out)) / len(set(exp)) if exp else 1.0 + ), } diff --git a/packages/evaluation/forge_eval/benchmark/swe_case.py b/packages/evaluation/forge_eval/benchmark/swe_case.py new file mode 100644 index 00000000..247b0aea --- /dev/null +++ b/packages/evaluation/forge_eval/benchmark/swe_case.py @@ -0,0 +1,51 @@ +"""Self-Eval Gate (F41) minted-case sandbox metadata. + +A minted Self-Eval Gate case is an ``agent_task`` :class:`~forge_eval.golden.GoldenCase` +auto-derived from one of the org's own merged PRs: the issue becomes the query, +and the regression check becomes a set of *hidden* fail-to-pass / pass-to-pass +tests replayed inside a sandbox — never surfaced to the model's context. Rather +than widen the shared, format-agnostic ``GoldenCase`` dataclass (used by every +other golden-set kind too), these sandbox fields are carried in +``GoldenCase.metadata`` and validated here. + +:func:`validate_freezable` (``forge_eval.benchmark.manifest``) already rejects +any ``agent_task`` case declaring ``expected_terminal_state == "merged"``; a +minted case must terminate no later than ``pr_opened``/``awaiting_review`` — +the harness never merges on the model's behalf (AC24, human approval gate). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from forge_eval.golden import GoldenCase + +__all__ = ["SweCaseFields", "parse_swe_case_fields"] + + +class SweCaseFields(BaseModel): + """Typed view of a minted Self-Eval Gate case's sandbox wiring. + + All fields are optional at the type level (a plain retrieval/task case has + none of them); a minted SWE case is expected to set every field. + """ + + #: Test node ids that must go fail -> pass after a correct resolution. + fail_to_pass: list[str] = Field(default_factory=list) + #: Test node ids that must stay passing (regression guard). + pass_to_pass: list[str] = Field(default_factory=list) + #: Sandbox base image/tag the case replays against. + sandbox_image: str | None = None + #: Ordered shell commands run once to prepare the sandbox before scoring. + setup_commands: list[str] = Field(default_factory=list) + #: Commit the sandbox checks out before applying any candidate change. + base_commit: str | None = None + + +def parse_swe_case_fields(case: GoldenCase) -> SweCaseFields: + """Extract + validate the Self-Eval Gate sandbox fields from ``case.metadata``. + + The hidden test ids never enter a model prompt: only the sandbox harness + reads ``fail_to_pass``/``pass_to_pass`` off the parsed result. + """ + return SweCaseFields.model_validate(case.metadata) diff --git a/packages/evaluation/forge_eval/mint/__init__.py b/packages/evaluation/forge_eval/mint/__init__.py new file mode 100644 index 00000000..64c2977d --- /dev/null +++ b/packages/evaluation/forge_eval/mint/__init__.py @@ -0,0 +1,39 @@ +"""Self-Eval Gate (F41) case minting. + +Auto-derives a *private, per-repo* regression benchmark case from one of the +org's own merged PRs: the PR's added/changed tests become the hidden +``fail_to_pass`` set (they fail on ``base_commit`` and pass after the merge), a +sample of untouched green tests becomes ``pass_to_pass``, and the sandbox +image/setup wire the whole thing to a reproducible harness. The hidden test ids +live only in ``GoldenCase.metadata`` (parsed via +:func:`forge_eval.benchmark.swe_case.parse_swe_case_fields`) and never enter a +model's context. + +Everything here is deterministic and offline: PR data arrives through the +:class:`PullRequestSource` protocol (a fake in tests, a GitHub adapter in prod) +and test outcomes through the :class:`TestRunner` protocol (the +:class:`GitWorktreeTestRunner` default is the ``SandboxKind.WORKTREE`` analogue — +a local git worktree + ``pytest`` subprocess, no network). +""" + +from __future__ import annotations + +from forge_eval.mint.pr_miner import ( + ChangedFile, + GitHubPullRequestSource, + GitWorktreeTestRunner, + PullRequestSource, + TestRunner, + changed_test_node_ids, + mint_case_from_pr, +) + +__all__ = [ + "ChangedFile", + "GitHubPullRequestSource", + "GitWorktreeTestRunner", + "PullRequestSource", + "TestRunner", + "changed_test_node_ids", + "mint_case_from_pr", +] diff --git a/packages/evaluation/forge_eval/mint/pr_miner.py b/packages/evaluation/forge_eval/mint/pr_miner.py new file mode 100644 index 00000000..d6d882fd --- /dev/null +++ b/packages/evaluation/forge_eval/mint/pr_miner.py @@ -0,0 +1,350 @@ +"""Mint a Self-Eval Gate benchmark case from a merged PR (F41). + +:func:`mint_case_from_pr` turns a merged :class:`~forge_contracts.PullRequest` +into a hidden fail-to-pass / pass-to-pass regression case: + +1. Parse the PR's changed-file *patches* for added/changed test node ids + (:func:`changed_test_node_ids`) — the ``fail_to_pass`` candidates. +2. Run those candidates at ``base_commit`` (before) and at the merge head + (after); keep the ones that go **fail -> pass** — a real regression the PR + fixed. +3. Sample a deterministic slice of *pre-existing* tests (untouched by the PR) + that are green at both refs — the ``pass_to_pass`` regression guard. +4. Emit an ``agent_task`` :class:`~forge_eval.golden.GoldenCase` whose + ``metadata`` carries the sandbox wiring + (:class:`~forge_eval.benchmark.swe_case.SweCaseFields`). + +PR data comes through :class:`PullRequestSource` and test outcomes through +:class:`TestRunner`, both injectable so the whole flow runs offline against a +fake GitHub + a local fixture repo. No live GitHub, no real provider. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +from forge_eval.benchmark.swe_case import parse_swe_case_fields +from forge_eval.golden import GoldenCase + +__all__ = [ + "ChangedFile", + "GitHubPullRequestSource", + "GitWorktreeTestRunner", + "PullRequestSource", + "TestRunner", + "changed_test_node_ids", + "mint_case_from_pr", +] + +#: A minted case must terminate no later than opening a PR — the harness never +#: merges on the model's behalf (AC24 human-approval gate). ``validate_freezable`` +#: rejects ``expected_terminal_state == "merged"``. +_TERMINAL_STATE = "pr_opened" + +#: Patterns that mark a path as a test module (pytest's default discovery). +_TEST_FILE_RE = re.compile(r"(^|/)(test_[^/]+|[^/]+_test)\.py$") +#: An added test def inside a unified-diff hunk (``+`` marker already stripped). +_DEF_RE = re.compile(r"^(?P\s*)(?:async\s+)?def\s+(?Ptest\w+)\s*\(") +#: A class header (context or added) inside a hunk, used to scope test methods. +_CLASS_RE = re.compile(r"^\s*class\s+(?PTest\w+)\b") + + +def _is_test_path(path: str) -> bool: + return bool(_TEST_FILE_RE.search(path)) + + +@dataclass(frozen=True) +class ChangedFile: + """One file touched by a PR, with its unified-diff ``patch`` (GitHub shape).""" + + path: str + status: str = "modified" # added | modified | removed | renamed + patch: str = "" + + +@runtime_checkable +class PullRequestSource(Protocol): + """PR data seam — a fake in tests, a GitHub adapter in prod. + + Kept minimal on purpose: the miner only needs the changed-file patches (to + find the added/changed tests) and the base commit (to run them "before"). + """ + + def pr_changed_files(self, repo: str, number: int) -> list[ChangedFile]: ... + + def pr_base_commit(self, repo: str, number: int) -> str: ... + + +@runtime_checkable +class TestRunner(Protocol): + """Runs/collects tests at a git ref. The offline sandbox seam. + + ``run_tests`` returns ``node_id -> passed`` for exactly the requested ids + (a missing/erroring test is reported ``False``). ``collect_tests`` lists the + node ids discoverable at ``ref`` (used to pick ``pass_to_pass`` candidates). + """ + + def run_tests(self, *, ref: str, node_ids: Sequence[str]) -> dict[str, bool]: ... + + def collect_tests(self, *, ref: str) -> list[str]: ... + + +def changed_test_node_ids(files: Sequence[ChangedFile]) -> list[str]: + """Extract pytest node ids for tests *added or changed* by a PR. + + Reads the ``+`` (added) lines of each test file's unified-diff ``patch`` and + emits ``path::test_name`` (or ``path::TestClass::test_name`` for a method + whose enclosing ``class Test*`` is visible in the same hunk). Deterministic, + de-duplicated, and order-stable across files. + """ + seen: set[str] = set() + node_ids: list[str] = [] + for changed in files: + if changed.status == "removed" or not _is_test_path(changed.path): + continue + current_class: str | None = None + for raw in changed.patch.splitlines(): + if raw.startswith(("+++", "---")) or raw.startswith("@@"): + current_class = None if raw.startswith("@@") else current_class + continue + marker, body = (raw[:1], raw[1:]) if raw[:1] in "+- " else ("", raw) + if marker == "-": # a removed line never contributes an added test + continue + class_match = _CLASS_RE.match(body) + if class_match: + current_class = class_match.group("name") + continue + if marker != "+": # only added lines yield new/changed tests + continue + def_match = _DEF_RE.match(body) + if not def_match: + continue + name = def_match.group("name") + if def_match.group("indent") and current_class: + node_id = f"{changed.path}::{current_class}::{name}" + else: + node_id = f"{changed.path}::{name}" + if node_id not in seen: + seen.add(node_id) + node_ids.append(node_id) + return node_ids + + +def _sample_pass_to_pass( + runner: TestRunner, + *, + base_commit: str, + head_ref: str, + excluded_files: set[str], + excluded_ids: set[str], + limit: int, +) -> list[str]: + """Deterministic sample of pre-existing tests that stay green base -> head.""" + if limit <= 0: + return [] + candidates = sorted( + node_id + for node_id in runner.collect_tests(ref=base_commit) + if node_id not in excluded_ids and node_id.split("::", 1)[0] not in excluded_files + )[: limit * 3] # over-sample; some may be flaky/absent at head + if not candidates: + return [] + base_ok = runner.run_tests(ref=base_commit, node_ids=candidates) + head_ok = runner.run_tests(ref=head_ref, node_ids=candidates) + green = [nid for nid in candidates if base_ok.get(nid) and head_ok.get(nid)] + return green[:limit] + + +def _default_case_id(repo: str, number: int) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", repo.lower()).strip("-") + return f"self-eval-{slug}-pr-{number}" + + +def mint_case_from_pr( + pr: Any, + repo: str, + *, + source: PullRequestSource, + runner: TestRunner, + sandbox_image: str | None = None, + setup_commands: Sequence[str] = (), + max_pass_to_pass: int = 10, + prompt: str | None = None, + case_id: str | None = None, +) -> GoldenCase | None: + """Derive a Self-Eval Gate :class:`GoldenCase` from a merged ``pr``. + + Returns ``None`` when the PR yields no ``fail_to_pass`` signal (no + added/changed test went fail -> pass) — such a PR can't seed a regression + case and is skipped rather than faked. + + ``pr`` is duck-typed on the frozen :class:`~forge_contracts.PullRequest` + surface (``.number``, ``.title``, ``.head_sha``). ``sandbox_image`` / + ``setup_commands`` describe the reproducible env the case replays in. + """ + number = int(pr.number) + head_ref = str(pr.head_sha) + base_commit = source.pr_base_commit(repo, number) + changed = source.pr_changed_files(repo, number) + + candidates = changed_test_node_ids(changed) + if not candidates: + return None + + base_outcomes = runner.run_tests(ref=base_commit, node_ids=candidates) + head_outcomes = runner.run_tests(ref=head_ref, node_ids=candidates) + fail_to_pass = sorted( + nid + for nid in candidates + if not base_outcomes.get(nid, False) and head_outcomes.get(nid, False) + ) + if not fail_to_pass: + return None + + changed_test_files = {c.path for c in changed if _is_test_path(c.path)} + pass_to_pass = _sample_pass_to_pass( + runner, + base_commit=base_commit, + head_ref=head_ref, + excluded_files=changed_test_files, + excluded_ids=set(candidates), + limit=max_pass_to_pass, + ) + + metadata: dict[str, Any] = { + "fail_to_pass": fail_to_pass, + "pass_to_pass": pass_to_pass, + "sandbox_image": sandbox_image, + "setup_commands": list(setup_commands), + "base_commit": base_commit, + "expected_terminal_state": _TERMINAL_STATE, + "source_repo": repo, + "source_pr": number, + "source_commit": head_ref, + } + case = GoldenCase( + id=case_id or _default_case_id(repo, number), + query=prompt or (getattr(pr, "title", None) or f"Resolve {repo}#{number}"), + expected_ids=fail_to_pass, + kind="agent_task", + tags=["self-eval"], + metadata=metadata, + ) + # Fail loudly if the sandbox wiring is malformed rather than shipping a case + # the harness can't replay. + parse_swe_case_fields(case) + return case + + +# --------------------------------------------------------------------------- # +# Concrete adapters # +# --------------------------------------------------------------------------- # + + +@dataclass +class GitHubPullRequestSource: + """:class:`PullRequestSource` backed by ``forge_integrations`` GitHub client. + + ``client`` is duck-typed on + :meth:`forge_integrations.github.GitHubClient.list_pr_files` / + :meth:`~forge_integrations.github.GitHubClient.pr_base_commit`, so this + module never imports ``forge_integrations`` (keeping ``forge_eval`` free of + that dependency); tests inject a fake implementing the same two methods. + """ + + client: Any + + def pr_changed_files(self, repo: str, number: int) -> list[ChangedFile]: + files: list[ChangedFile] = [] + for raw in self.client.list_pr_files(repo, number): + files.append( + ChangedFile( + path=str(raw.get("filename") or raw.get("path") or ""), + status=str(raw.get("status") or "modified"), + patch=str(raw.get("patch") or ""), + ) + ) + return [f for f in files if f.path] + + def pr_base_commit(self, repo: str, number: int) -> str: + return str(self.client.pr_base_commit(repo, number)) + + +@dataclass +class GitWorktreeTestRunner: + """Offline ``TestRunner`` — a git worktree + ``pytest`` subprocess per ref. + + The ``SandboxKind.WORKTREE`` analogue used by tests and the V1 minter: for a + ref it detaches a throwaway worktree of ``repo_path`` at that commit and runs + ``pytest`` there. Fully local, no network. A node id absent/erroring at a ref + is reported ``False`` (via pytest's non-zero exit), which is exactly the + "test didn't exist yet on base_commit" signal the miner keys fail-to-pass on. + """ + + repo_path: str | Path + pytest_args: Sequence[str] = field(default_factory=lambda: ("-p", "no:cacheprovider")) + timeout_s: int = 300 + + def _git(self, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(self.repo_path), *args], + capture_output=True, + text=True, + timeout=self.timeout_s, + check=False, + ) + + def _pytest(self, worktree: str, extra: Sequence[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "pytest", *self.pytest_args, *extra], + cwd=worktree, + capture_output=True, + text=True, + timeout=self.timeout_s, + check=False, + ) + + def _with_worktree(self, ref: str) -> tuple[str, Callable[[], None]]: + tmp = tempfile.mkdtemp(prefix="forge-selfeval-") + added = self._git("worktree", "add", "--detach", "--force", tmp, ref) + if added.returncode != 0: + shutil.rmtree(tmp, ignore_errors=True) + raise RuntimeError(f"git worktree add {ref!r} failed: {added.stderr.strip()}") + + def cleanup() -> None: + self._git("worktree", "remove", "--force", tmp) + shutil.rmtree(tmp, ignore_errors=True) + + return tmp, cleanup + + def run_tests(self, *, ref: str, node_ids: Sequence[str]) -> dict[str, bool]: + if not node_ids: + return {} + worktree, cleanup = self._with_worktree(ref) + try: + # Run each node id in isolation so a collection error on one absent + # test never taints the verdict for the others. + return {nid: self._pytest(worktree, [nid, "-q"]).returncode == 0 for nid in node_ids} + finally: + cleanup() + + def collect_tests(self, *, ref: str) -> list[str]: + worktree, cleanup = self._with_worktree(ref) + try: + proc = self._pytest(worktree, ["--collect-only", "-q"]) + node_ids: list[str] = [] + for line in proc.stdout.splitlines(): + candidate = line.strip() + if "::" in candidate and not candidate.startswith(("<", "warning")): + node_ids.append(candidate) + return node_ids + finally: + cleanup() diff --git a/packages/evaluation/forge_eval/sweval/__init__.py b/packages/evaluation/forge_eval/sweval/__init__.py new file mode 100644 index 00000000..169fa469 --- /dev/null +++ b/packages/evaluation/forge_eval/sweval/__init__.py @@ -0,0 +1,23 @@ +"""Self-Eval Gate (F41) sandboxed fail-to-pass / pass-to-pass runner. + +A minted Self-Eval Gate case carries HIDDEN tests (``fail_to_pass`` / +``pass_to_pass``) that never enter a model's context. :func:`run_swe_case` +applies a candidate patch inside a sandbox, runs those hidden tests through the +:class:`~forge_contracts.sandbox.SandboxSession` execution seam, and scores the +resolution rate — the ground-truth signal the Self-Eval Gate blocks on. +""" + +from __future__ import annotations + +from forge_eval.sweval.gate import SelfEvalGate, SelfEvalRegressionError +from forge_eval.sweval.runner import SweCaseResult, run_swe_case +from forge_eval.sweval.self_eval import SelfEvalScorecard, run_self_eval + +__all__ = [ + "SelfEvalGate", + "SelfEvalRegressionError", + "SelfEvalScorecard", + "SweCaseResult", + "run_self_eval", + "run_swe_case", +] diff --git a/packages/evaluation/forge_eval/sweval/gate.py b/packages/evaluation/forge_eval/sweval/gate.py new file mode 100644 index 00000000..7824f02f --- /dev/null +++ b/packages/evaluation/forge_eval/sweval/gate.py @@ -0,0 +1,63 @@ +"""The Self-Eval Gate: block a config change that regresses on the private suite. + +A model/prompt/router config change is refused if, evaluated against the +workspace's private per-repo suite, its resolution rate drops below the recorded +baseline. The eval runner and baseline lookup are injected (production wires the +agent runtime + sandbox + a persisted baseline; tests inject a fake), and the +gate is a no-op on cold start (no baseline / no private suite) so existing config +flows stay green until a suite exists. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from forge_eval.sweval.self_eval import SelfEvalScorecard + +#: Run the private-suite Self-Eval for a proposed config; None = no private suite. +EvalRunner = Callable[[UUID, Any], Awaitable[SelfEvalScorecard | None]] +#: Recorded baseline resolution rate for a workspace; None = no baseline yet. +BaselineLookup = Callable[[UUID], float | None] + + +class SelfEvalRegressionError(Exception): + """Raised when a proposed config regresses the private-suite resolution rate.""" + + def __init__(self, *, scorecard: SelfEvalScorecard, baseline_rate: float) -> None: + self.scorecard = scorecard + self.baseline_rate = baseline_rate + super().__init__( + f"Self-Eval regression: resolution rate " + f"{scorecard.resolution_rate:.3f} < baseline {baseline_rate:.3f} " + f"({scorecard.resolved}/{scorecard.total} cases resolved)" + ) + + +@dataclass(frozen=True) +class SelfEvalGate: + """Gate a config change on the private-suite resolution rate.""" + + eval_runner: EvalRunner + baseline_for: BaselineLookup + + async def check_config( + self, workspace_id: UUID, proposed_config: Any, *, force: bool = False + ) -> SelfEvalScorecard | None: + """Raise :class:`SelfEvalRegressionError` if ``proposed_config`` regresses. + + Returns the passing scorecard (or None on cold start / forced override). + """ + if force: + return None + baseline = self.baseline_for(workspace_id) + if baseline is None: + return None # cold start — no baseline to regress against + scorecard = await self.eval_runner(workspace_id, proposed_config) + if scorecard is None: + return None # no private suite for this workspace + if not scorecard.meets(baseline): + raise SelfEvalRegressionError(scorecard=scorecard, baseline_rate=baseline) + return scorecard diff --git a/packages/evaluation/forge_eval/sweval/runner.py b/packages/evaluation/forge_eval/sweval/runner.py new file mode 100644 index 00000000..da89013c --- /dev/null +++ b/packages/evaluation/forge_eval/sweval/runner.py @@ -0,0 +1,154 @@ +"""Sandboxed fail-to-pass / pass-to-pass runner for the Self-Eval Gate. + +The runner is the ground-truth executor: it takes a minted case (with hidden +fail-to-pass + pass-to-pass tests), applies a candidate patch produced by a +``solve_fn``, and runs the hidden tests inside a real +:class:`~forge_contracts.sandbox.SandboxSession` (the local ``worktree`` +provider offline; a stronger isolation kind in production). Nothing about the +hidden tests is ever handed to ``solve_fn`` — it only sees the case's public +prompt/metadata — so a model cannot game the gate by reading the checks. + +``solve_fn(case) -> Mapping[path, content]`` returns the files the candidate +change writes into the worktree (a file-write patch; simple + deterministic to +verify). A case is *resolved* only when every ``fail_to_pass`` test now passes +AND no ``pass_to_pass`` test regresses. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path + +from forge_contracts import SandboxKind, SandboxSpec +from forge_contracts.sandbox import SandboxProvider, SandboxSession +from forge_eval.benchmark.swe_case import SweCaseFields, parse_swe_case_fields +from forge_eval.golden import GoldenCase + +#: A candidate solution as file writes (relative path -> new content). +SolveFn = Callable[[GoldenCase], Mapping[str, str]] + +_DEFAULT_TIMEOUT_S = 600 +_EVIDENCE_CAP = 4096 +#: Case-metadata keys that must NEVER reach solve_fn (the model) — the hidden +#: tests the gate scores against. +_HIDDEN_KEYS = ("fail_to_pass", "pass_to_pass") + + +@dataclass(frozen=True) +class SweCaseResult: + """The outcome of running one minted case against a candidate patch.""" + + case_id: str + #: The fail-to-pass tests that PASS after the patch — used as ``output_ids`` + #: so the ``agent.fail_to_pass_rate`` metric (set overlap vs the case's + #: ``expected_ids`` = all fail-to-pass) scores the resolution rate. + output_ids: list[str] + #: pass-to-pass tests that REGRESSED (were green, now fail) — any regression + #: fails the case even if every fail-to-pass now passes. + regressed: list[str] = field(default_factory=list) + resolved: bool = False + evidence: str = "" + + +def _apply_patch(worktree: Path, files: Mapping[str, str]) -> None: + """Write the candidate patch's files into the worktree (path-traversal safe).""" + root = worktree.resolve() + for rel, content in files.items(): + dest = (root / rel).resolve() + if not dest.is_relative_to(root): + raise ValueError(f"patch path escapes the worktree: {rel!r}") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(content, encoding="utf-8") + + +def _test_command(test_id: str) -> str: + """pytest node-id / path -> a deterministic, quiet invocation.""" + return f"python -m pytest -q -p no:cacheprovider {test_id}" + + +async def run_swe_case( + *, + case: GoldenCase, + solve_fn: SolveFn, + sandbox_provider: SandboxProvider, + worktree_path: str, + timeout_s: int = _DEFAULT_TIMEOUT_S, +) -> SweCaseResult: + """Apply ``solve_fn``'s patch and run the case's hidden tests in a sandbox. + + ``worktree_path`` is a prepared checkout at the case's ``base_commit`` + (the caller/miner sets it up); the runner writes the patch, runs + ``setup_commands``, then the hidden fail-to-pass + pass-to-pass tests. + """ + fields: SweCaseFields = parse_swe_case_fields(case) + if not fields.fail_to_pass: + raise ValueError(f"case {case.id!r} has no fail_to_pass tests to gate on") + + # solve_fn only ever sees a REDACTED case — the hidden test ids are stripped + # so they can never enter a model's context (the harness reads them itself). + public_case = replace( + case, + metadata={k: v for k, v in case.metadata.items() if k not in _HIDDEN_KEYS}, + ) + patch = solve_fn(public_case) + _apply_patch(Path(worktree_path), patch) + + spec = SandboxSpec( + agent_run_id=uuid.uuid4(), + workspace_id=uuid.uuid4(), + kind=SandboxKind.WORKTREE, + host_worktree_path=worktree_path, + exec_timeout_seconds=timeout_s, + ) + await sandbox_provider.preflight() + session: SandboxSession = await sandbox_provider.create(spec) + evidence_parts: list[str] = [] + try: + for cmd in fields.setup_commands: + out = await session.run(cmd, cwd=session.workspace_dir, timeout_s=timeout_s) + if out.exit_code != 0: + evidence_parts.append(f"setup failed: {cmd}\n{out.stdout}{out.stderr}") + return SweCaseResult( + case_id=case.id, + output_ids=[], + resolved=False, + evidence="\n".join(evidence_parts)[-_EVIDENCE_CAP:], + ) + + passed = await _run_tests(session, fields.fail_to_pass, timeout_s, evidence_parts) + regressed = [ + t + for t in fields.pass_to_pass + if t not in await _run_tests(session, [t], timeout_s, evidence_parts) + ] + resolved = set(passed) == set(fields.fail_to_pass) and not regressed + return SweCaseResult( + case_id=case.id, + output_ids=passed, + regressed=regressed, + resolved=resolved, + evidence="\n".join(evidence_parts)[-_EVIDENCE_CAP:], + ) + finally: + await session.teardown(reason="self_eval_complete") + + +async def _run_tests( + session: SandboxSession, + test_ids: Sequence[str], + timeout_s: int, + evidence: list[str], +) -> list[str]: + """Return the subset of ``test_ids`` that PASS (exit 0) in the sandbox.""" + passing: list[str] = [] + for test_id in test_ids: + out = await session.run( + _test_command(test_id), cwd=session.workspace_dir, timeout_s=timeout_s + ) + if out.exit_code == 0: + passing.append(test_id) + else: + evidence.append(f"{test_id}: exit {out.exit_code}\n{out.stdout}{out.stderr}") + return passing diff --git a/packages/evaluation/forge_eval/sweval/self_eval.py b/packages/evaluation/forge_eval/sweval/self_eval.py new file mode 100644 index 00000000..5ebe37f2 --- /dev/null +++ b/packages/evaluation/forge_eval/sweval/self_eval.py @@ -0,0 +1,59 @@ +"""Aggregate a Self-Eval run over a workspace's private benchmark suite. + +Runs every minted case through :func:`run_swe_case` with the candidate config's +``solve_fn`` and reports the resolution rate — the ground-truth signal the +Self-Eval Gate blocks a model/prompt/router change on. Pure and injectable: +``solve_fn`` and the per-case worktree factory are passed in, so the API layer +wires the real agent runtime + sandbox while tests use a scripted model. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field + +from forge_contracts.sandbox import SandboxProvider +from forge_eval.golden import GoldenCase +from forge_eval.sweval.runner import SolveFn, SweCaseResult, run_swe_case + +#: Prepare (or look up) a worktree checked out at the case's base_commit and +#: return its host path. +WorktreeFactory = Callable[[GoldenCase], str] + + +@dataclass(frozen=True) +class SelfEvalScorecard: + """The result of a Self-Eval run over a private suite.""" + + total: int + resolved: int + resolution_rate: float + per_case: list[SweCaseResult] = field(default_factory=list) + + def meets(self, baseline_rate: float) -> bool: + """True iff this run does NOT regress vs a recorded baseline rate.""" + return self.resolution_rate >= baseline_rate + + +async def run_self_eval( + *, + cases: Sequence[GoldenCase], + solve_fn: SolveFn, + sandbox_provider: SandboxProvider, + worktree_for: WorktreeFactory, +) -> SelfEvalScorecard: + """Score ``solve_fn`` (the config under test) over every case in the suite.""" + results: list[SweCaseResult] = [] + for case in cases: + results.append( + await run_swe_case( + case=case, + solve_fn=solve_fn, + sandbox_provider=sandbox_provider, + worktree_path=worktree_for(case), + ) + ) + total = len(results) + resolved = sum(1 for r in results if r.resolved) + rate = resolved / total if total else 1.0 + return SelfEvalScorecard(total=total, resolved=resolved, resolution_rate=rate, per_case=results) diff --git a/packages/evaluation/tests/benchmark/test_swe_case.py b/packages/evaluation/tests/benchmark/test_swe_case.py new file mode 100644 index 00000000..66fa8af2 --- /dev/null +++ b/packages/evaluation/tests/benchmark/test_swe_case.py @@ -0,0 +1,73 @@ +"""F41 unit tests — minted Self-Eval Gate case sandbox metadata.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from forge_eval.benchmark import ( + BenchmarkFrozenError, + SweCaseFields, + parse_swe_case_fields, + validate_freezable, +) +from forge_eval.golden import GoldenCase + + +def _minted_case(**metadata_overrides) -> GoldenCase: + metadata = { + "fail_to_pass": ["tests/test_auth.py::test_refresh"], + "pass_to_pass": ["tests/test_auth.py::test_login"], + "sandbox_image": "forge/sandbox:py3.14", + "setup_commands": ["uv sync"], + "base_commit": "abc123", + "expected_terminal_state": "pr_opened", + } + metadata.update(metadata_overrides) + return GoldenCase( + id="issue-42", + query="Fix token refresh race", + expected_ids=["ISSUE-42"], + kind="agent_task", + metadata=metadata, + ) + + +def test_parse_swe_case_fields_roundtrip() -> None: + case = _minted_case() + fields = parse_swe_case_fields(case) + assert fields == SweCaseFields( + fail_to_pass=["tests/test_auth.py::test_refresh"], + pass_to_pass=["tests/test_auth.py::test_login"], + sandbox_image="forge/sandbox:py3.14", + setup_commands=["uv sync"], + base_commit="abc123", + ) + + +def test_parse_swe_case_fields_defaults_on_empty_metadata() -> None: + case = GoldenCase(id="x", query="q", expected_ids=["y"]) + fields = parse_swe_case_fields(case) + assert fields.fail_to_pass == [] + assert fields.pass_to_pass == [] + assert fields.sandbox_image is None + assert fields.setup_commands == [] + assert fields.base_commit is None + + +def test_parse_swe_case_fields_rejects_wrong_types() -> None: + case = _minted_case(fail_to_pass="not-a-list") + with pytest.raises(ValidationError): + parse_swe_case_fields(case) + + +def test_minted_case_at_pr_opened_is_freezable() -> None: + """AC24: a minted case terminating at pr_opened (not merged) may freeze.""" + validate_freezable([_minted_case()]) # does not raise + + +def test_minted_case_declaring_merged_is_rejected() -> None: + """AC24: hidden tests never let a minted case terminate at merged.""" + bad = _minted_case(expected_terminal_state="merged") + with pytest.raises(BenchmarkFrozenError, match="merged"): + validate_freezable([bad]) diff --git a/packages/evaluation/tests/mint/test_pr_miner.py b/packages/evaluation/tests/mint/test_pr_miner.py new file mode 100644 index 00000000..11be5c38 --- /dev/null +++ b/packages/evaluation/tests/mint/test_pr_miner.py @@ -0,0 +1,201 @@ +"""F41 Self-Eval Gate minting — offline (fake GitHub + real fixture git repo). + +The GitHub side is a fake :class:`FakePullRequestSource` (no network); the test +side is the *real* :class:`GitWorktreeTestRunner` driving ``pytest`` in a git +worktree of a tiny fixture repo. The fixture PR both adds ``mul`` to ``calc.py`` +and adds ``test_mul.py`` — so the new test fails on ``base_commit`` (``mul`` does +not exist) and passes on the merge head: a genuine fail -> pass regression. +""" + +from __future__ import annotations + +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from forge_eval.benchmark import parse_swe_case_fields +from forge_eval.mint import ChangedFile, GitWorktreeTestRunner, changed_test_node_ids +from forge_eval.mint.pr_miner import mint_case_from_pr + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="git required for worktrees") + + +@dataclass +class FakePR: + """Duck-typed on the frozen forge_contracts.PullRequest surface.""" + + number: int + head_sha: str + title: str = "Add multiply to calc" + repo: str = "acme/widgets" + + +class FakePullRequestSource: + """In-memory PR-data source — the fake GitHub the miner reads through.""" + + def __init__(self, *, base_commit: str, changed_files: list[ChangedFile]) -> None: + self._base = base_commit + self._files = changed_files + + def pr_changed_files(self, repo: str, number: int) -> list[ChangedFile]: + return list(self._files) + + def pr_base_commit(self, repo: str, number: int) -> str: + return self._base + + +def _git(repo: Path, *args: str) -> str: + out = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=True, + ) + return out.stdout.strip() + + +def _write(repo: Path, rel: str, body: str) -> None: + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + +@pytest.fixture +def fixture_repo(tmp_path: Path) -> tuple[Path, str, str]: + """A 2-commit git repo: base (no ``mul``) -> merge (adds ``mul`` + its test). + + Returns ``(repo_path, base_sha, head_sha)``. + """ + repo = tmp_path / "widgets" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "t@example.com") + _git(repo, "config", "user.name", "Tester") + + # --- base commit: calc.add + a pre-existing green test (pass_to_pass) --- + _write(repo, "calc.py", "def add(a, b):\n return a + b\n") + _write( + repo, + "test_add.py", + "from calc import add\n\n\ndef test_add():\n assert add(1, 2) == 3\n", + ) + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "base") + base_sha = _git(repo, "rev-parse", "HEAD") + + # --- merge head: PR adds calc.mul AND test_mul.py (fail_to_pass) --- + _write( + repo, + "calc.py", + "def add(a, b):\n return a + b\n\n\ndef mul(a, b):\n return a * b\n", + ) + _write( + repo, + "test_mul.py", + "from calc import mul\n\n\ndef test_mul():\n assert mul(2, 3) == 6\n", + ) + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "add mul") + head_sha = _git(repo, "rev-parse", "HEAD") + + return repo, base_sha, head_sha + + +_MUL_PATCH = ( + "@@ -0,0 +1,4 @@\n+from calc import mul\n+\n+\n+def test_mul():\n+ assert mul(2, 3) == 6\n" +) + + +def test_changed_test_node_ids_parses_added_defs() -> None: + files = [ + ChangedFile(path="test_mul.py", status="added", patch=_MUL_PATCH), + ChangedFile(path="calc.py", status="modified", patch="@@ -1 +1,4 @@\n+def mul(a, b):\n"), + ] + # Only the test file contributes; the source-file def is ignored. + assert changed_test_node_ids(files) == ["test_mul.py::test_mul"] + + +def test_changed_test_node_ids_scopes_class_methods() -> None: + patch = ( + "@@ -0,0 +1,3 @@\n class TestThing:\n+ def test_method(self):\n+ assert True\n" + ) + files = [ChangedFile(path="tests/test_thing.py", status="modified", patch=patch)] + assert changed_test_node_ids(files) == ["tests/test_thing.py::TestThing::test_method"] + + +def test_mint_case_from_pr_derives_fail_to_pass(fixture_repo: tuple[Path, str, str]) -> None: + repo, base_sha, head_sha = fixture_repo + source = FakePullRequestSource( + base_commit=base_sha, + changed_files=[ + ChangedFile(path="test_mul.py", status="added", patch=_MUL_PATCH), + ChangedFile( + path="calc.py", + status="modified", + patch="@@ -1 +1,4 @@\n+def mul(a, b):\n+ return a * b\n", + ), + ], + ) + runner = GitWorktreeTestRunner(repo_path=repo) + pr = FakePR(number=7, head_sha=head_sha) + + case = mint_case_from_pr( + pr, + "acme/widgets", + source=source, + runner=runner, + sandbox_image="python:3.14-slim", + setup_commands=["pip install -e ."], + ) + + assert case is not None + assert case.id == "self-eval-acme-widgets-pr-7" + assert case.kind == "agent_task" + assert case.expected_ids == ["test_mul.py::test_mul"] + + fields = parse_swe_case_fields(case) + assert fields.fail_to_pass == ["test_mul.py::test_mul"] + # The pre-existing green test is sampled as a regression guard. + assert "test_add.py::test_add" in fields.pass_to_pass + # The new test itself is never a pass_to_pass entry. + assert "test_mul.py::test_mul" not in fields.pass_to_pass + assert fields.base_commit == base_sha + assert fields.sandbox_image == "python:3.14-slim" + assert fields.setup_commands == ["pip install -e ."] + # Never declares a merge terminal state (AC24 human-approval gate). + assert case.metadata["expected_terminal_state"] != "merged" + + +def test_mint_returns_none_when_no_regression_signal(fixture_repo: tuple[Path, str, str]) -> None: + repo, base_sha, head_sha = fixture_repo + # A PR that added a test which *already passes* on base yields no fail->pass. + already_green = ( + "@@ -0,0 +1,4 @@\n" + "+from calc import add\n" + "+\n" + "+def test_add_again():\n" + "+ assert add(2, 2) == 4\n" + ) + source = FakePullRequestSource( + base_commit=base_sha, + changed_files=[ChangedFile(path="test_extra.py", status="added", patch=already_green)], + ) + # test_extra.py does not exist on either ref of the fixture, so the runner + # reports it failing on base AND head -> not a fail->pass -> no case. + runner = GitWorktreeTestRunner(repo_path=repo) + pr = FakePR(number=8, head_sha=head_sha) + assert mint_case_from_pr(pr, "acme/widgets", source=source, runner=runner) is None + + +def test_mint_returns_none_when_no_tests_changed(fixture_repo: tuple[Path, str, str]) -> None: + repo, base_sha, head_sha = fixture_repo + source = FakePullRequestSource( + base_commit=base_sha, + changed_files=[ChangedFile(path="README.md", status="modified", patch="@@ +1 @@\n+docs\n")], + ) + runner = GitWorktreeTestRunner(repo_path=repo) + pr = FakePR(number=9, head_sha=head_sha) + assert mint_case_from_pr(pr, "acme/widgets", source=source, runner=runner) is None diff --git a/packages/evaluation/tests/sweval/test_self_eval.py b/packages/evaluation/tests/sweval/test_self_eval.py new file mode 100644 index 00000000..ddb0171e --- /dev/null +++ b/packages/evaluation/tests/sweval/test_self_eval.py @@ -0,0 +1,72 @@ +"""Offline tests for the Self-Eval run aggregation over a private suite.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from forge_agent.sandbox import LocalSandboxProvider +from forge_eval.golden import GoldenCase +from forge_eval.sweval import run_self_eval + +_BROKEN = "def add(a, b):\n return a - b\n" +_FIXED = "def add(a, b):\n return a + b\n" +_TEST = "from mymod import add\n\n\ndef test_add():\n assert add(2, 3) == 5\n" +_FTP = "test_mymod.py::test_add" + + +def _make_worktree(root: Path, name: str) -> Path: + wt = root / name + wt.mkdir() + (wt / "mymod.py").write_text(_BROKEN, encoding="utf-8") + (wt / "test_mymod.py").write_text(_TEST, encoding="utf-8") + return wt + + +def _case(cid: str) -> GoldenCase: + return GoldenCase( + id=cid, + query="make add correct", + expected_ids=[_FTP], + kind="agent_task", + metadata={"fail_to_pass": [_FTP], "pass_to_pass": []}, + ) + + +@pytest.mark.asyncio +async def test_all_resolved_gives_full_rate(tmp_path: Path) -> None: + cases = [_case("a"), _case("b")] + worktrees = {c.id: _make_worktree(tmp_path, c.id) for c in cases} + card = await run_self_eval( + cases=cases, + solve_fn=lambda _c: {"mymod.py": _FIXED}, + sandbox_provider=LocalSandboxProvider(), + worktree_for=lambda c: str(worktrees[c.id]), + ) + assert card.total == 2 + assert card.resolved == 2 + assert card.resolution_rate == 1.0 + assert card.meets(1.0) is True + + +@pytest.mark.asyncio +async def test_regression_lowers_rate_and_fails_baseline(tmp_path: Path) -> None: + cases = [_case("a"), _case("b")] + worktrees = {c.id: _make_worktree(tmp_path, c.id) for c in cases} + + # A config that only solves case "a" (leaves "b" broken). case.id survives + # redaction (only the hidden-test metadata is stripped). + def solve(case: GoldenCase) -> dict[str, str]: + return {"mymod.py": _FIXED} if case.id == "a" else {"mymod.py": _BROKEN} + + card = await run_self_eval( + cases=cases, + solve_fn=solve, + sandbox_provider=LocalSandboxProvider(), + worktree_for=lambda c: str(worktrees[c.id]), + ) + assert card.resolved == 1 + assert card.resolution_rate == 0.5 + assert card.meets(1.0) is False # regressed vs a 100% baseline + assert card.meets(0.5) is True diff --git a/packages/evaluation/tests/sweval/test_self_eval_gate.py b/packages/evaluation/tests/sweval/test_self_eval_gate.py new file mode 100644 index 00000000..0e7f6b60 --- /dev/null +++ b/packages/evaluation/tests/sweval/test_self_eval_gate.py @@ -0,0 +1,52 @@ +"""Tests for the Self-Eval Gate blocking logic (pure, no sandbox).""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +import pytest + +from forge_eval.sweval import SelfEvalGate, SelfEvalRegressionError, SelfEvalScorecard + +WS = uuid4() + + +def _card(rate: float) -> SelfEvalScorecard: + return SelfEvalScorecard(total=10, resolved=int(rate * 10), resolution_rate=rate) + + +def _gate(*, rate: float | None, baseline: float | None) -> SelfEvalGate: + async def runner(_ws: UUID, _cfg: object) -> SelfEvalScorecard | None: + return _card(rate) if rate is not None else None + + return SelfEvalGate(eval_runner=runner, baseline_for=lambda _ws: baseline) + + +@pytest.mark.asyncio +async def test_regression_is_blocked() -> None: + gate = _gate(rate=0.6, baseline=0.9) + with pytest.raises(SelfEvalRegressionError): + await gate.check_config(WS, {"model": "cheap"}) + + +@pytest.mark.asyncio +async def test_equal_or_better_is_allowed() -> None: + assert (await _gate(rate=0.9, baseline=0.9).check_config(WS, {})).resolution_rate == 0.9 + assert (await _gate(rate=0.95, baseline=0.9).check_config(WS, {})).resolution_rate == 0.95 + + +@pytest.mark.asyncio +async def test_cold_start_no_baseline_is_noop() -> None: + assert await _gate(rate=0.1, baseline=None).check_config(WS, {}) is None + + +@pytest.mark.asyncio +async def test_no_private_suite_is_noop() -> None: + assert await _gate(rate=None, baseline=0.9).check_config(WS, {}) is None + + +@pytest.mark.asyncio +async def test_force_overrides_the_gate() -> None: + # A regressing config passes when forced (the caller audits the override). + gate = _gate(rate=0.1, baseline=0.9) + assert await gate.check_config(WS, {}, force=True) is None diff --git a/packages/evaluation/tests/sweval/test_swe_runner.py b/packages/evaluation/tests/sweval/test_swe_runner.py new file mode 100644 index 00000000..8b09edad --- /dev/null +++ b/packages/evaluation/tests/sweval/test_swe_runner.py @@ -0,0 +1,101 @@ +"""Offline tests for the Self-Eval Gate sandboxed runner. + +Uses the local ``worktree`` sandbox provider on a temp checkout — no network, +no live model. A candidate patch that fixes the module resolves the case; a +wrong patch does not; hidden tests are never handed to ``solve_fn``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from forge_agent.sandbox import LocalSandboxProvider +from forge_eval.golden import GoldenCase +from forge_eval.sweval import run_swe_case + +_BROKEN = "def add(a, b):\n return a - b\n" +_FIXED = "def add(a, b):\n return a + b\n" +_TEST = "from mymod import add\n\n\ndef test_add():\n assert add(2, 3) == 5\n" +_KEEP = "def test_keep():\n assert True\n" + +_FTP = "test_mymod.py::test_add" +_PTP = "test_keep.py::test_keep" + + +@pytest.fixture +def worktree(tmp_path: Path) -> Path: + (tmp_path / "mymod.py").write_text(_BROKEN, encoding="utf-8") + (tmp_path / "test_mymod.py").write_text(_TEST, encoding="utf-8") + (tmp_path / "test_keep.py").write_text(_KEEP, encoding="utf-8") + return tmp_path + + +def _case(*, fail: list[str], keep: list[str] | None = None) -> GoldenCase: + return GoldenCase( + id="swe-1", + query="make add() correct", + expected_ids=list(fail), + kind="agent_task", + metadata={"fail_to_pass": list(fail), "pass_to_pass": list(keep or [])}, + ) + + +@pytest.mark.asyncio +async def test_correct_patch_resolves(worktree: Path) -> None: + res = await run_swe_case( + case=_case(fail=[_FTP], keep=[_PTP]), + solve_fn=lambda _c: {"mymod.py": _FIXED}, + sandbox_provider=LocalSandboxProvider(), + worktree_path=str(worktree), + ) + assert res.resolved is True + assert res.output_ids == [_FTP] + assert res.regressed == [] + + +@pytest.mark.asyncio +async def test_wrong_patch_does_not_resolve(worktree: Path) -> None: + res = await run_swe_case( + case=_case(fail=[_FTP]), + solve_fn=lambda _c: {"mymod.py": _BROKEN}, # still wrong + sandbox_provider=LocalSandboxProvider(), + worktree_path=str(worktree), + ) + assert res.resolved is False + assert res.output_ids == [] + + +@pytest.mark.asyncio +async def test_pass_to_pass_regression_fails_the_case(worktree: Path) -> None: + # A patch that fixes fail-to-pass but breaks a pass-to-pass test must NOT resolve. + res = await run_swe_case( + case=_case(fail=[_FTP], keep=[_PTP]), + solve_fn=lambda _c: { + "mymod.py": _FIXED, + "test_keep.py": "def test_keep():\n assert False\n", + }, + sandbox_provider=LocalSandboxProvider(), + worktree_path=str(worktree), + ) + assert res.resolved is False + assert res.regressed == [_PTP] + + +@pytest.mark.asyncio +async def test_solve_fn_never_sees_hidden_tests(worktree: Path) -> None: + seen: dict[str, object] = {} + + def solve(case: GoldenCase) -> dict[str, str]: + seen["metadata"] = dict(case.metadata) + return {"mymod.py": _FIXED} + + await run_swe_case( + case=_case(fail=[_FTP], keep=[_PTP]), + solve_fn=solve, + sandbox_provider=LocalSandboxProvider(), + worktree_path=str(worktree), + ) + assert "fail_to_pass" not in seen["metadata"] # type: ignore[operator] + assert "pass_to_pass" not in seen["metadata"] # type: ignore[operator] diff --git a/packages/integration-sdk/forge_integrations/github.py b/packages/integration-sdk/forge_integrations/github.py index 0e59e2aa..9c2fba3d 100644 --- a/packages/integration-sdk/forge_integrations/github.py +++ b/packages/integration-sdk/forge_integrations/github.py @@ -705,6 +705,46 @@ def list_reviews(self, pr: PullRequest) -> list[Review]: ) ] + def list_pr_files(self, repo: str, number: int) -> list[dict[str, Any]]: + """Paginated read of ``GET /repos/{r}/pulls/{n}/files`` (raw file objects). + + Each item carries ``filename``/``status``/``patch`` — the unified-diff + the Self-Eval Gate miner (F41) parses for added/changed test node ids. + Returned as raw dicts so this SDK never depends on ``forge_eval``. + """ + r = self.owner_repo(repo) + return list( + self._paginate( + "GET", + f"/repos/{r}/pulls/{number}/files", + params={"per_page": 100}, + action="list_pr_files", + repo=r, + ) + ) + + def pr_base_commit(self, repo: str, number: int) -> str: + """Return a PR's base commit sha (``base.sha`` of ``GET .../pulls/{n}``). + + The "before" ref the Self-Eval Gate miner (F41) replays added tests + against to confirm they fail prior to the merge. + """ + return self._pr_ref_sha(repo, number, "base") + + def pr_head_commit(self, repo: str, number: int) -> str: + """Return a PR's head commit sha (``head.sha`` of ``GET .../pulls/{n}``). + + The "after" ref (the merged change) the Self-Eval Gate miner (F41) + replays added tests against to confirm they now pass. + """ + return self._pr_ref_sha(repo, number, "head") + + def _pr_ref_sha(self, repo: str, number: int, side: str) -> str: + r = self.owner_repo(repo) + resp = self._request("GET", f"/repos/{r}/pulls/{number}", action="get_pr", repo=r) + self._raise_for_status(resp) + return str((resp.json().get(side) or {}).get("sha") or "") + def close_pr(self, pr: PullRequest) -> PullRequest: """Close a PR (``PATCH .../pulls/{n}`` with ``state=closed``).""" r = self.owner_repo(pr.repo) diff --git a/uv.lock b/uv.lock index 39580929..7487d674 100644 --- a/uv.lock +++ b/uv.lock @@ -1319,6 +1319,8 @@ dependencies = [ { name = "forge-contracts" }, { name = "forge-db" }, { name = "forge-deploy" }, + { name = "forge-eval" }, + { name = "forge-integrations" }, { name = "forge-knowledge" }, { name = "forge-marketplace" }, { name = "forge-mcp" }, @@ -1338,6 +1340,8 @@ requires-dist = [ { name = "forge-contracts", editable = "packages/contracts" }, { name = "forge-db", editable = "packages/db" }, { name = "forge-deploy", editable = "packages/deploy-core" }, + { name = "forge-eval", editable = "packages/evaluation" }, + { name = "forge-integrations", editable = "packages/integration-sdk" }, { name = "forge-knowledge", editable = "packages/knowledge-core" }, { name = "forge-marketplace", editable = "packages/marketplace-sdk" }, { name = "forge-mcp", editable = "packages/mcp-sdk" }, @@ -1423,7 +1427,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/93/43e116ee114b28737ba7e12952a0d4e2f55944d0f84e42bc91ba7192a3c9/greenlet-3.5.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:fd2e02fa07485778536a036222d616ab957b1d533f36b3ed98ce725d9c9d3117", size = 288202, upload-time = "2026-06-26T18:23:49.604Z" }, { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" }, { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" }, + { url = "https://files.pythonhosted.org/packages/86/a9/73fa62893d5b84b4205544e6b673c654cc43aa5b9899bac00f04d64af73d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8d19fe6c39ebff9259f07bcc685d3290f8fa4ea2278e51dd0008e4d6b0f2d814", size = 670657, upload-time = "2026-06-26T19:24:19.967Z" }, { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" }, + { url = "https://files.pythonhosted.org/packages/29/7e/2ffce64929fb3cab7b65d5a0b20aaf9764e227681d731b041077fc9a525a/greenlet-3.5.3-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:962c5df2db8cb446da51edf1ca5296c389d93b99c9d8aa2ee4c7d0d8f1218260", size = 473497, upload-time = "2026-06-26T19:25:39.421Z" }, { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" }, { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" }, { url = "https://files.pythonhosted.org/packages/7a/94/91aec0030bea75c4b3244251d0de60a1f3432d1ecb53ab6c437fb5c3ba61/greenlet-3.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:7669aa24cf2a1041d6f7899575b494a3ab4cf68bfcc8609b1dc0be7272db835e", size = 240754, upload-time = "2026-06-26T18:22:15.669Z" }, @@ -1431,14 +1437,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/95/3e161213d7f1d378d15aa9e792093e9bfe01844680d04b7fd6e0107c9098/greenlet-3.5.3-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:271a8ea7c1024e8a0d7dd2be66dd66dda8a07193f41a17b9e924f7600f5b62be", size = 296389, upload-time = "2026-06-26T18:22:20.657Z" }, { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" }, { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/8faec206b851c22b1733545fda900829a1f3f5b1c78ae7e0fb3dba57d9f4/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b897d97759425953f69a9c0fac67f8fe333ec0ce7377ef186fb2b0c3ad5e354d", size = 659582, upload-time = "2026-06-26T19:24:21.357Z" }, { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" }, + { url = "https://files.pythonhosted.org/packages/b4/55/50c19e49f8045834ada71ef12f8ad048eba8517c6aa41161bed676328fae/greenlet-3.5.3-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:3236754d423955ea08e9bb5f6c04a7895f9e22c290b66aa7653fcb922d839eb0", size = 491037, upload-time = "2026-06-26T19:25:40.672Z" }, { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" }, { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" }, { url = "https://files.pythonhosted.org/packages/e4/15/7edb977e08f9bff702fe42d6c902702786ff6b9694058b4e6a2a6ac90e57/greenlet-3.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:efc6bd60ea02e085862c74a3ef64b147ffc6f1a5ea7d9f26e7a939943f68c1e3", size = 243626, upload-time = "2026-06-26T18:24:41.485Z" }, { url = "https://files.pythonhosted.org/packages/2c/8a/93928dce91e6b3598b5e779e8d1fd6576a504640c58e78627077f6a7a91a/greenlet-3.5.3-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:ea03f2f04367845d6b58eeed276e1e56e51f0b97d8ad5a88a7d20a91dc9056cc", size = 288860, upload-time = "2026-06-26T18:22:48.07Z" }, { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" }, { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/25/aa/952cf28c2ff949a8c971134fb43854dd7eaa737218723aaef758f8c9aead/greenlet-3.5.3-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cefa9cef4b371f9844c6053db71f1138bc6807bab1578b0dae5149c1f1141357", size = 674261, upload-time = "2026-06-26T19:24:22.79Z" }, { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f2/b00d6f5e63e531a93562b2ec1a4c320fbee91f580fc42e6417af69d706e5/greenlet-3.5.3-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:6219b6d04dbf6ba6084d77dc609e8473060dc55f759cbf626d512122781fa128", size = 480322, upload-time = "2026-06-26T19:25:41.852Z" }, { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" }, { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" }, { url = "https://files.pythonhosted.org/packages/68/5d/00d469daae3c65d2bf620b10eee82eb022127d483c6bc8c69fae6f3fbf17/greenlet-3.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:dd99329bbc15ca78dcc583dba05d0b1b0bae01ab6c2174989f5aaee3e41ac930", size = 241027, upload-time = "2026-06-26T18:22:38.203Z" }, @@ -1446,7 +1456,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/da/4f4a8450962fad137c1c8981a3f1b8919d06c829993d4d476f9c525d5173/greenlet-3.5.3-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:176bc16a721fa5fc294d70b87b4dfa5fbdd251b3da5d5372735ecef9bd7d6d0c", size = 297221, upload-time = "2026-06-26T18:23:27.176Z" }, { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" }, { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/e9/39/0e0938a75115b939d42733a2a12e1d349653c9531fe6fe563e8a681f04e6/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d192579ed281051396dddd7f7754dac6259e6b1fb26378c87b66622f8e3f91", size = 663706, upload-time = "2026-06-26T19:24:24.312Z" }, { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/5b/41/35d1c678cdb3c3b9e6bee691728e563cfb294202b23c7a4c3c2ccc343589/greenlet-3.5.3-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:4399eb8d041f20b68d943918bc55502a93d6fdc0a37c14da7881c04139acee9d", size = 498803, upload-time = "2026-06-26T19:25:43.063Z" }, { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" }, { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" }, { url = "https://files.pythonhosted.org/packages/a7/13/1055e1dda7882073eda533e2b96c62e55bbd2db7fda6d5ece992febc7071/greenlet-3.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:8ff8bed3e3baa20a3ea261ce00526f1898ad4801d4886fd2220580ee0ad8fadf", size = 244007, upload-time = "2026-06-26T18:22:04.353Z" },