diff --git a/apps/api/forge_api/services/self_eval_service.py b/apps/api/forge_api/services/self_eval_service.py new file mode 100644 index 00000000..b8c8e69a --- /dev/null +++ b/apps/api/forge_api/services/self_eval_service.py @@ -0,0 +1,126 @@ +"""Persistence for the Self-Eval Gate baseline (F41). + +The Self-Eval Gate refuses a model/prompt/router config change if, re-evaluated +on a workspace's private per-repo suite, its resolution rate drops below a frozen +baseline. This service owns that baseline: reading it (the gate's +``baseline_for`` lookup) and writing it when a run establishes or an admin +promotes one. It is deliberately storage-only — the *policy* of when to promote +a baseline lives in the run/enforcement layer, and callers must pass an already +redacted ``config`` snapshot (no secrets ever land in this table). +""" + +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker + +from forge_db.models.benchmark import SelfEvalBaseline + + +@dataclass(frozen=True) +class BaselineRecord: + """A workspace's frozen Self-Eval baseline for one private suite.""" + + workspace_id: uuid.UUID + benchmark_suite_id: uuid.UUID + baseline_rate: float + resolved: int + total: int + + +class SelfEvalService: + """Read/write the per-(workspace, suite) Self-Eval baseline resolution rate.""" + + def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + self._session_factory = session_factory + + def workspace_baseline(self, workspace_id: uuid.UUID) -> float | None: + """The rate a config change in this workspace must not fall below. + + Cold start (no baseline recorded yet) returns ``None`` so the gate is a + no-op. If several private suites in the workspace carry a baseline, the + most recently updated one wins — this is the value bound as the gate's + ``baseline_for`` lookup. + """ + with self._session_factory() as session: + stmt = ( + select(SelfEvalBaseline.baseline_rate) + .where(SelfEvalBaseline.workspace_id == workspace_id) + .order_by(SelfEvalBaseline.updated_at.desc()) + ) + return session.scalars(stmt).first() + + def baseline_for_suite( + self, workspace_id: uuid.UUID, benchmark_suite_id: uuid.UUID + ) -> BaselineRecord | None: + """The full baseline record for one (workspace, suite), or ``None``.""" + with self._session_factory() as session: + row = self._get(session, workspace_id, benchmark_suite_id) + return _to_record(row) if row is not None else None + + def record_baseline( + self, + *, + workspace_id: uuid.UUID, + benchmark_suite_id: uuid.UUID, + resolved: int, + total: int, + resolution_rate: float, + config: Mapping[str, Any], + recorded_by: uuid.UUID | None = None, + overwrite: bool = True, + ) -> BaselineRecord: + """Upsert the baseline for (workspace, suite); ``config`` must be redacted. + + With ``overwrite=False`` an existing baseline is left untouched (cold-start + establish semantics) and its current value is returned — so a regressing + run can never silently *lower* the bar it is meant to defend. + """ + with self._session_factory() as session: + row = self._get(session, workspace_id, benchmark_suite_id) + if row is None: + row = SelfEvalBaseline( + workspace_id=workspace_id, + benchmark_suite_id=benchmark_suite_id, + baseline_rate=resolution_rate, + resolved=resolved, + total=total, + config=dict(config), + recorded_by=recorded_by, + ) + session.add(row) + elif overwrite: + row.baseline_rate = resolution_rate + row.resolved = resolved + row.total = total + row.config = dict(config) + row.recorded_by = recorded_by + session.commit() + session.refresh(row) + return _to_record(row) + + @staticmethod + def _get( + session: Session, workspace_id: uuid.UUID, benchmark_suite_id: uuid.UUID + ) -> SelfEvalBaseline | None: + return session.scalars( + select(SelfEvalBaseline).where( + SelfEvalBaseline.workspace_id == workspace_id, + SelfEvalBaseline.benchmark_suite_id == benchmark_suite_id, + ) + ).one_or_none() + + +def _to_record(row: SelfEvalBaseline) -> BaselineRecord: + return BaselineRecord( + workspace_id=row.workspace_id, + benchmark_suite_id=row.benchmark_suite_id, + baseline_rate=row.baseline_rate, + resolved=row.resolved, + total=row.total, + ) diff --git a/apps/api/tests/test_self_eval_service.py b/apps/api/tests/test_self_eval_service.py new file mode 100644 index 00000000..d19abcd6 --- /dev/null +++ b/apps/api/tests/test_self_eval_service.py @@ -0,0 +1,128 @@ +"""Tests for the Self-Eval baseline persistence service (F41).""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator + +import pytest +from sqlalchemy import StaticPool, create_engine +from sqlalchemy.orm import Session, sessionmaker + +from forge_api.services.self_eval_service import SelfEvalService +from forge_db.base import Base + +WS = uuid.uuid4() +OTHER_WS = uuid.uuid4() +SUITE = uuid.uuid4() +OTHER_SUITE = uuid.uuid4() + + +@pytest.fixture +def session_factory() -> Iterator[sessionmaker[Session]]: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + Base.metadata.create_all(engine) + try: + yield sessionmaker(bind=engine, expire_on_commit=False, class_=Session) + finally: + Base.metadata.drop_all(engine) + engine.dispose() + + +@pytest.fixture +def service(session_factory: sessionmaker[Session]) -> SelfEvalService: + return SelfEvalService(session_factory=session_factory) + + +def test_cold_start_returns_none(service: SelfEvalService) -> None: + assert service.workspace_baseline(WS) is None + assert service.baseline_for_suite(WS, SUITE) is None + + +def test_record_then_read_back(service: SelfEvalService) -> None: + rec = service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=8, + total=10, + resolution_rate=0.8, + config={"model": "claude-opus"}, + ) + assert rec.baseline_rate == 0.8 + assert service.workspace_baseline(WS) == 0.8 + full = service.baseline_for_suite(WS, SUITE) + assert full is not None + assert (full.resolved, full.total) == (8, 10) + + +def test_upsert_overwrites_same_suite(service: SelfEvalService) -> None: + service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=7, + total=10, + resolution_rate=0.7, + config={}, + ) + service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=9, + total=10, + resolution_rate=0.9, + config={}, + ) + assert service.workspace_baseline(WS) == 0.9 # one row per (workspace, suite) + + +def test_overwrite_false_preserves_existing(service: SelfEvalService) -> None: + service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=9, + total=10, + resolution_rate=0.9, + config={}, + ) + # A later, worse run must never lower the bar it defends. + rec = service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=4, + total=10, + resolution_rate=0.4, + config={}, + overwrite=False, + ) + assert rec.baseline_rate == 0.9 + assert service.workspace_baseline(WS) == 0.9 + + +def test_baselines_are_workspace_isolated(service: SelfEvalService) -> None: + service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=9, + total=10, + resolution_rate=0.9, + config={}, + ) + assert service.workspace_baseline(OTHER_WS) is None + + +def test_config_is_stored_verbatim_but_isolated(service: SelfEvalService) -> None: + # The service persists what it is given; the caller is responsible for + # passing an already-redacted config. Confirm it round-trips as a copy. + original = {"model": "claude-opus", "effort": "high"} + service.record_baseline( + workspace_id=WS, + benchmark_suite_id=SUITE, + resolved=10, + total=10, + resolution_rate=1.0, + config=original, + ) + original["model"] = "mutated" # must not affect the stored snapshot + assert service.workspace_baseline(WS) == 1.0 diff --git a/apps/worker/forge_worker/self_eval_run.py b/apps/worker/forge_worker/self_eval_run.py new file mode 100644 index 00000000..8c34a72f --- /dev/null +++ b/apps/worker/forge_worker/self_eval_run.py @@ -0,0 +1,273 @@ +"""Production Self-Eval runner (A3): drive the real agent runtime + sandbox. + +This is the live implementation of the injected ``EvalRunner`` seam the Self-Eval +Gate (``forge_eval.sweval.gate``) blocks on. It lives in the worker because this +is one of the only layers that may depend on BOTH ``forge_agent`` (to run a +coding agent) and ``forge_eval`` (the sandboxed scoring engine) — keeping +``forge_eval`` itself free of any agent-runtime dependency. + +For a workspace's private per-repo suite it: + +1. resolves the suite (on-disk case dir + a local git clone to check out from); +2. for each minted case, checks out ``base_commit`` into a scratch worktree, + runs a coding :class:`~forge_agent.AgentRunner` (the config under test) with + repo read/write tools, and diffs the worktree into a file-write patch — the + ``SolveFn`` the sandboxed runner re-applies before running the HIDDEN tests; +3. scores the resolution rate via :func:`forge_eval.sweval.run_self_eval`. + +Cold start is honest: no private suite, no minted cases, or no resolvable model +client (offline / no BYOK) all return ``None``, so the gate no-ops rather than +fabricating a score. +""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from forge_agent import AgentRunner, ToolRegistry +from forge_agent.tools import ToolResult +from forge_contracts import AgentObjective, ModelClient +from forge_contracts.sandbox import SandboxProvider +from forge_eval.benchmark.manifest import load_manifest +from forge_eval.benchmark.swe_case import parse_swe_case_fields +from forge_eval.golden import GoldenCase +from forge_eval.sweval import SelfEvalScorecard, run_self_eval + +__all__ = [ + "ProductionEvalRunner", + "SelfEvalSuiteHandle", + "agent_solve", + "build_coder_tools", +] + +#: Tool policy actions the eval agent is scoped to (allow-list on the objective). +_READ = "read_repo" +_WRITE = "write_code" + + +def _resolve_under(root: Path, raw: str) -> Path: + """Resolve ``raw`` under ``root``, rejecting traversal outside it.""" + candidate = (root / raw).resolve() + root_resolved = root.resolve() + if root_resolved != candidate and root_resolved not in candidate.parents: + raise ValueError(f"path escapes worktree: {raw!r}") + return candidate + + +def build_coder_tools(worktree: Path) -> ToolRegistry: + """A minimal repo-editing tool set (read/write/list) scoped under ``worktree``.""" + registry = ToolRegistry() + + def read_file(args: dict[str, Any]) -> ToolResult: + path = _resolve_under(worktree, str(args.get("path", ""))) + if not path.is_file(): + return ToolResult(ok=False, error=f"not a file: {args.get('path')}") + return ToolResult(ok=True, output=path.read_text()) + + def write_file(args: dict[str, Any]) -> ToolResult: + rel = str(args.get("path", "")).strip() + if not rel: + return ToolResult(ok=False, error="write_file requires a 'path'") + target = _resolve_under(worktree, rel) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(str(args.get("content", ""))) + return ToolResult(ok=True, output=f"wrote {rel}") + + def list_files(args: dict[str, Any]) -> ToolResult: + base = _resolve_under(worktree, str(args.get("path", "."))) + if not base.is_dir(): + return ToolResult(ok=False, error=f"not a directory: {args.get('path')}") + names = sorted( + p.relative_to(worktree).as_posix() + for p in base.rglob("*") + if p.is_file() and ".git" not in p.parts + ) + return ToolResult(ok=True, output="\n".join(names)) + + registry.add("read_file", read_file, action=_READ, description="Read a repo file") + registry.add("write_file", write_file, action=_WRITE, description="Create or overwrite a file") + registry.add("list_files", list_files, action=_READ, description="List repo files under a path") + return registry + + +def _git(repo_path: str, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", repo_path, *args], + check=check, + capture_output=True, + text=True, + ) + + +def _add_worktree(repo_path: str, base_commit: str) -> Path: + """Check out ``base_commit`` into a fresh detached worktree; return its path. + + ``git worktree add`` must create the leaf itself, so the checkout goes in a + ``wt`` subdir of a throwaway parent (removed wholesale by ``_remove_worktree``). + """ + parent = Path(tempfile.mkdtemp(prefix="forge-selfeval-")) + worktree = parent / "wt" + _git(repo_path, "worktree", "add", "--detach", str(worktree), base_commit) + return worktree + + +def _remove_worktree(repo_path: str, worktree: Path) -> None: + _git(repo_path, "worktree", "remove", "--force", str(worktree), check=False) + shutil.rmtree(worktree.parent, ignore_errors=True) + + +def _changed_files(worktree: Path) -> dict[str, str]: + """Every added/modified/renamed file in ``worktree`` vs its base, as a patch map.""" + _git(str(worktree), "add", "-A", check=False) + diff = _git( + str(worktree), + "diff", + "--cached", + "--name-only", + "--diff-filter=ACMR", + check=False, + ) + patch: dict[str, str] = {} + for rel in diff.stdout.splitlines(): + rel = rel.strip() + if not rel: + continue + path = worktree / rel + if path.is_file(): + patch[rel] = path.read_text() + return patch + + +def agent_solve( + case: GoldenCase, + *, + model_client: ModelClient, + repo_path: str, + model: str | None = None, + max_iterations: int = 12, +) -> dict[str, str]: + """Run the coding agent over ``case`` in a scratch worktree; return its edits. + + The agent only ever sees the case's public ``query`` (the hidden tests are + stripped by :func:`forge_eval.sweval.run_swe_case` before this is called), + edits a throwaway checkout at the case's ``base_commit`` through the + read/write tools, and the diff of that checkout is the patch the sandboxed + runner then applies before scoring against the hidden tests. + """ + fields = parse_swe_case_fields(case) + base = fields.base_commit or "HEAD" + scratch = _add_worktree(repo_path, base) + try: + tools = build_coder_tools(scratch) + objective = AgentObjective( + objective=case.query, + instructions=( + "Modify the repository so the described change is fully implemented. " + "Read files with read_file/list_files and save every edit with write_file. " + "Do not add or edit test files — only the implementation." + ), + allowed_actions=[_READ, _WRITE], + model=model, + context={"self_eval_case_id": case.id}, + ) + runner = AgentRunner( + model_client, + tools=tools, + repo_root=str(scratch), + max_iterations=max_iterations, + ) + runner.run(objective) + return _changed_files(scratch) + finally: + _remove_worktree(repo_path, scratch) + + +@dataclass(frozen=True) +class SelfEvalSuiteHandle: + """Where a workspace's private suite lives: its case dir + a git clone.""" + + benchmark_suite_id: uuid.UUID + #: On-disk suite version directory (holds ``manifest.yaml`` + minted cases). + version_dir: str + #: Local git clone of the suite's source repo, to check out ``base_commit`` from. + repo_path: str + + +#: Resolve the private suite for a workspace; ``None`` = no suite (cold start). +SuiteResolver = Callable[[uuid.UUID], SelfEvalSuiteHandle | None] +#: Build the model client for a proposed config; ``None`` = offline / no BYOK. +ModelClientFor = Callable[[Any], ModelClient | None] + + +@dataclass(frozen=True) +class ProductionEvalRunner: + """The live ``EvalRunner`` the Self-Eval Gate blocks on. + + Satisfies ``Callable[[UUID, Any], Awaitable[SelfEvalScorecard | None]]``: + given a workspace and a proposed config, it runs the config's agent over the + workspace's private suite and returns the scorecard, or ``None`` on any + cold-start condition (no suite / no cases / no resolvable model client). + """ + + resolve_suite: SuiteResolver + model_client_for: ModelClientFor + sandbox_provider: SandboxProvider + max_iterations: int = 12 + + async def __call__( + self, workspace_id: uuid.UUID, proposed_config: Any + ) -> SelfEvalScorecard | None: + handle = self.resolve_suite(workspace_id) + if handle is None: + return None # no private suite for this workspace + model_client = self.model_client_for(proposed_config) + if model_client is None: + return None # offline / no BYOK — never fabricate a score + _scoring, cases = load_manifest(Path(handle.version_dir)) + if not cases: + return None # suite has no minted cases yet + model = _config_model(proposed_config) + + worktrees: list[Path] = [] + + def worktree_for(case: GoldenCase) -> str: + fields = parse_swe_case_fields(case) + worktree = _add_worktree(handle.repo_path, fields.base_commit or "HEAD") + worktrees.append(worktree) + return str(worktree) + + def solve_fn(case: GoldenCase) -> dict[str, str]: + return agent_solve( + case, + model_client=model_client, + repo_path=handle.repo_path, + model=model, + max_iterations=self.max_iterations, + ) + + try: + return await run_self_eval( + cases=cases, + solve_fn=solve_fn, + sandbox_provider=self.sandbox_provider, + worktree_for=worktree_for, + ) + finally: + for worktree in worktrees: + _remove_worktree(handle.repo_path, worktree) + + +def _config_model(proposed_config: Any) -> str | None: + """Best-effort extract of a model name from a proposed config (dict or attr).""" + if isinstance(proposed_config, dict): + model = proposed_config.get("model") + return str(model) if model else None + model = getattr(proposed_config, "model", None) + return str(model) if model else None diff --git a/apps/worker/tests/test_self_eval_run.py b/apps/worker/tests/test_self_eval_run.py new file mode 100644 index 00000000..4b487e8d --- /dev/null +++ b/apps/worker/tests/test_self_eval_run.py @@ -0,0 +1,249 @@ +"""Offline tests for the production Self-Eval runner (A3). + +No network, no live model: a scripted model client authors the fix through the +coder tools, real git worktrees are checked out from a temp repo, and the +hidden tests run in the local ``worktree`` sandbox. Proves the agent -> file-map +adapter and the end-to-end scorecard without any BYOK credentials. +""" + +from __future__ import annotations + +import subprocess +import uuid +from pathlib import Path + +import pytest +import yaml + +from forge_agent.sandbox import LocalSandboxProvider +from forge_agent.testing import ScriptedModelClient, finish_response, tool_response +from forge_eval.golden import GoldenCase +from forge_worker.self_eval_run import ( + ProductionEvalRunner, + SelfEvalSuiteHandle, + agent_solve, + build_coder_tools, +) + +_BROKEN = "def add(a, b):\n return a - b\n" +_FIXED = "def add(a, b):\n return a + b\n" +_TEST = "from mymod import add\n\n\ndef test_add():\n assert add(2, 3) == 5\n" +_KEEP = "def test_keep():\n assert True\n" + +_FTP = "test_mymod.py::test_add" +_PTP = "test_keep.py::test_keep" + + +def _init_repo(root: Path) -> str: + """Create a git repo with the broken module + hidden tests; return base sha.""" + (root / "mymod.py").write_text(_BROKEN, encoding="utf-8") + (root / "test_mymod.py").write_text(_TEST, encoding="utf-8") + (root / "test_keep.py").write_text(_KEEP, encoding="utf-8") + + def git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(root), *args], check=True, capture_output=True, text=True + ).stdout.strip() + + git("init", "-q") + git("config", "user.email", "selfeval@forge.test") + git("config", "user.name", "Self Eval") + git("add", "-A") + git("commit", "-q", "-m", "base") + return git("rev-parse", "HEAD") + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + root.mkdir() + _init_repo(root) + return root + + +def _scripted_writer(path: str, content: str) -> ScriptedModelClient: + """A model that writes one file then finishes.""" + return ScriptedModelClient( + responses=[tool_response("write_file", {"path": path, "content": content})], + default=finish_response("done", confidence=0.9), + ) + + +# --------------------------------------------------------------------------- # +# Coder tools # +# --------------------------------------------------------------------------- # + + +def _run_tool(tools, name: str, args: dict) -> object: + tool = tools.get(name) + assert tool is not None, f"tool {name!r} not registered" + return tool.run(args) + + +def test_coder_tools_read_write_list(tmp_path: Path) -> None: + tools = build_coder_tools(tmp_path) + (tmp_path / "a.py").write_text("hello", encoding="utf-8") + + read = _run_tool(tools, "read_file", {"path": "a.py"}) + assert read.ok and read.output == "hello" + + wrote = _run_tool(tools, "write_file", {"path": "sub/b.py", "content": "x = 1"}) + assert wrote.ok + assert (tmp_path / "sub" / "b.py").read_text() == "x = 1" + + listed = _run_tool(tools, "list_files", {"path": "."}) + assert "a.py" in listed.output and "sub/b.py" in listed.output + + +def test_coder_tools_reject_path_traversal(tmp_path: Path) -> None: + tools = build_coder_tools(tmp_path) + with pytest.raises(ValueError, match="escapes worktree"): + _run_tool(tools, "write_file", {"path": "../evil.py", "content": "nope"}) + + +# --------------------------------------------------------------------------- # +# agent_solve — the agent -> file-map adapter # +# --------------------------------------------------------------------------- # + + +def _case(base: str) -> GoldenCase: + return GoldenCase( + id="swe-1", + query="make add() correct", + expected_ids=[_FTP], + kind="agent_task", + metadata={"fail_to_pass": [_FTP], "pass_to_pass": [_PTP], "base_commit": base}, + ) + + +def test_agent_solve_returns_agent_edits(repo: Path) -> None: + base = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, check=True + ).stdout.strip() + patch = agent_solve( + _case(base), + model_client=_scripted_writer("mymod.py", _FIXED), + repo_path=str(repo), + ) + assert patch == {"mymod.py": _FIXED} + + +def test_agent_solve_empty_when_agent_edits_nothing(repo: Path) -> None: + base = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, check=True + ).stdout.strip() + # A model that just finishes without writing produces no patch (honest miss). + idle = ScriptedModelClient(responses=[], default=finish_response("nothing to do")) + assert agent_solve(_case(base), model_client=idle, repo_path=str(repo)) == {} + + +# --------------------------------------------------------------------------- # +# ProductionEvalRunner # +# --------------------------------------------------------------------------- # + + +def _write_suite(version_dir: Path, base: str) -> None: + """A minimal non-frozen suite dir with one minted SWE case.""" + version_dir.mkdir(parents=True) + (version_dir / "cases").mkdir() + (version_dir / "cases" / "self_eval.json").write_text( + __import__("json").dumps( + [ + { + "id": "swe-1", + "query": "make add() correct", + "expected_ids": [_FTP], + "kind": "agent_task", + "metadata": { + "fail_to_pass": [_FTP], + "pass_to_pass": [_PTP], + "base_commit": base, + }, + } + ] + ), + encoding="utf-8", + ) + (version_dir / "manifest.yaml").write_text( + yaml.safe_dump( + { + "slug": "self-eval", + "version": "1.0.0", + "title": "Private self-eval suite", + "schema_version": 1, + "frozen": False, + "scoring": { + "primary_metric": "agent.fail_to_pass_rate", + "metric_weights": {"agent.fail_to_pass_rate": 1.0}, + }, + "case_files": ["cases/self_eval.json"], + } + ), + encoding="utf-8", + ) + + +@pytest.mark.asyncio +async def test_runner_cold_start_no_suite() -> None: + runner = ProductionEvalRunner( + resolve_suite=lambda _ws: None, + model_client_for=lambda _cfg: _scripted_writer("mymod.py", _FIXED), + sandbox_provider=LocalSandboxProvider(), + ) + assert await runner(uuid.uuid4(), {"model": "x"}) is None + + +@pytest.mark.asyncio +async def test_runner_offline_no_model(repo: Path, tmp_path: Path) -> None: + version_dir = tmp_path / "suite" + base = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, check=True + ).stdout.strip() + _write_suite(version_dir, base) + handle = SelfEvalSuiteHandle(uuid.uuid4(), str(version_dir), str(repo)) + runner = ProductionEvalRunner( + resolve_suite=lambda _ws: handle, + model_client_for=lambda _cfg: None, # offline / no BYOK + sandbox_provider=LocalSandboxProvider(), + ) + assert await runner(uuid.uuid4(), {"model": "x"}) is None + + +@pytest.mark.asyncio +async def test_runner_scores_end_to_end(repo: Path, tmp_path: Path) -> None: + version_dir = tmp_path / "suite" + base = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, check=True + ).stdout.strip() + _write_suite(version_dir, base) + handle = SelfEvalSuiteHandle(uuid.uuid4(), str(version_dir), str(repo)) + runner = ProductionEvalRunner( + resolve_suite=lambda _ws: handle, + model_client_for=lambda _cfg: _scripted_writer("mymod.py", _FIXED), + sandbox_provider=LocalSandboxProvider(), + ) + card = await runner(uuid.uuid4(), {"model": "claude-opus"}) + assert card is not None + assert (card.total, card.resolved) == (1, 1) + assert card.resolution_rate == 1.0 + + +@pytest.mark.asyncio +async def test_runner_wrong_config_does_not_resolve(repo: Path, tmp_path: Path) -> None: + version_dir = tmp_path / "suite" + base = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, check=True + ).stdout.strip() + _write_suite(version_dir, base) + handle = SelfEvalSuiteHandle(uuid.uuid4(), str(version_dir), str(repo)) + # A config whose agent writes a still-broken module must score 0 — the signal + # the gate blocks a regressing config on. + runner = ProductionEvalRunner( + resolve_suite=lambda _ws: handle, + model_client_for=lambda _cfg: _scripted_writer("mymod.py", _BROKEN), + sandbox_provider=LocalSandboxProvider(), + ) + card = await runner(uuid.uuid4(), {"model": "cheap"}) + assert card is not None + assert (card.total, card.resolved) == (1, 0) + assert card.resolution_rate == 0.0 diff --git a/packages/db/forge_db/models/__init__.py b/packages/db/forge_db/models/__init__.py index 5e0108a7..80f800d1 100644 --- a/packages/db/forge_db/models/__init__.py +++ b/packages/db/forge_db/models/__init__.py @@ -12,7 +12,7 @@ from forge_db.models.attestation import Attestation from forge_db.models.audit import AuditChainHead, AuditLog, AuditLogImmutableError from forge_db.models.automation import AutomationExecution, AutomationRule -from forge_db.models.benchmark import BenchmarkSubmission, BenchmarkSuite +from forge_db.models.benchmark import BenchmarkSubmission, BenchmarkSuite, SelfEvalBaseline from forge_db.models.connections import MCPConnection, RepositoryConnection from forge_db.models.cost import CostEvent, ModelPrice from forge_db.models.deployment import ( @@ -307,6 +307,7 @@ "ScimToken", "ScopeType", "Secret", + "SelfEvalBaseline", "SkillProfile", "SkillProfileSnapshot", "SpecDocument", diff --git a/packages/db/forge_db/models/benchmark.py b/packages/db/forge_db/models/benchmark.py index 8ee81149..a733c4e7 100644 --- a/packages/db/forge_db/models/benchmark.py +++ b/packages/db/forge_db/models/benchmark.py @@ -176,4 +176,48 @@ class BenchmarkSubmission(ForgeModel): ) -__all__ = ["BenchmarkSubmission", "BenchmarkSuite"] +class SelfEvalBaseline(ForgeModel): + """The recorded baseline resolution rate a config change is gated against. + + Self-Eval Gate (F41): when a workspace's private per-repo suite is first run + (or an admin accepts a better result), the resolution rate is frozen here as + the baseline. A later model/prompt/router change is refused if, re-evaluated + on the same suite, it scores below ``baseline_rate``. Exactly one baseline + per (workspace, suite) — a new run upserts this row. + """ + + __tablename__ = "self_eval_baseline" + __table_args__ = ( + UniqueConstraint( + "workspace_id", + "benchmark_suite_id", + name="uq_self_eval_baseline_workspace_suite", + ), + Index("ix_self_eval_baseline_workspace", "workspace_id"), + ) + + #: The workspace whose private suite this baseline belongs to (NOT NULL — + #: a baseline is always tenant-scoped, unlike a global benchmark suite). + workspace_id: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("workspace.id", ondelete="CASCADE"), + nullable=False, + ) + benchmark_suite_id: Mapped[uuid.UUID] = mapped_column( + Uuid(as_uuid=True), + ForeignKey("benchmark_suite.id", ondelete="CASCADE"), + nullable=False, + ) + #: Frozen resolution rate (0..1) — the value a config change must not fall below. + baseline_rate: Mapped[float] = mapped_column(Float, nullable=False) + #: Scorecard provenance for the baseline run (resolved of total cases). + resolved: Mapped[int] = mapped_column(Integer, nullable=False) + total: Mapped[int] = mapped_column(Integer, nullable=False) + #: Redacted config snapshot that established this baseline (no secrets). + config: Mapped[dict[str, Any]] = mapped_column(json_type(), default=dict, nullable=False) + recorded_by: Mapped[uuid.UUID | None] = mapped_column( + Uuid(as_uuid=True), ForeignKey("app_user.id", ondelete="SET NULL"), nullable=True + ) + + +__all__ = ["BenchmarkSubmission", "BenchmarkSuite", "SelfEvalBaseline"] diff --git a/packages/db/migrations/versions/0001_baseline.py b/packages/db/migrations/versions/0001_baseline.py index a1739e67..a7e77d10 100644 --- a/packages/db/migrations/versions/0001_baseline.py +++ b/packages/db/migrations/versions/0001_baseline.py @@ -75,6 +75,9 @@ # F35 benchmark-leaderboard — created by 0018_f35_benchmark_leaderboard. "benchmark_suite", "benchmark_submission", + # F41 Self-Eval Gate baseline (FK -> benchmark_suite, itself deferred) — + # created by 0040_self_eval_baseline. + "self_eval_baseline", # F01 board persistence (task dependency adjacency) — created by # 0024_board_persistence. "task_dependency", diff --git a/packages/db/migrations/versions/0040_self_eval_baseline.py b/packages/db/migrations/versions/0040_self_eval_baseline.py new file mode 100644 index 00000000..7eb76dcc --- /dev/null +++ b/packages/db/migrations/versions/0040_self_eval_baseline.py @@ -0,0 +1,98 @@ +"""self-eval gate: per-(workspace, suite) baseline resolution rate + +Adds the ``self_eval_baseline`` table (``forge_db.models.benchmark. +SelfEvalBaseline``): the frozen resolution rate a later model/prompt/router +change is gated against. Exactly one baseline per (workspace, suite) — a new +run upserts the row — so the Self-Eval Gate can look up "the rate this config +must not fall below" for a workspace's private per-repo suite. + +Purely additive: a brand-new table, no change to any existing table, so it +cannot alter current behaviour. Idempotent like 0036-0039: ``upgrade`` creates +the table only if absent; ``downgrade`` drops only what this revision adds. + +Revision ID: 0040_self_eval_baseline +Revises: 0039_self_eval_suite_scoping +Create Date: 2026-07-14 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +import forge_db.models # noqa: F401 (registers all models on Base.metadata) + +# revision identifiers, used by Alembic. +revision: str = "0040_self_eval_baseline" +down_revision: str | None = "0039_self_eval_suite_scoping" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_TABLE = "self_eval_baseline" +_INDEX_NAME = "ix_self_eval_baseline_workspace" +_UNIQUE_NAME = "uq_self_eval_baseline_workspace_suite" + + +def _existing_tables() -> set[str]: + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def upgrade() -> None: + if _TABLE in _existing_tables(): + return + op.create_table( + _TABLE, + sa.Column("id", sa.Uuid(as_uuid=True), primary_key=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "workspace_id", + sa.Uuid(as_uuid=True), + sa.ForeignKey("workspace.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "benchmark_suite_id", + sa.Uuid(as_uuid=True), + sa.ForeignKey("benchmark_suite.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("baseline_rate", sa.Float(), nullable=False), + sa.Column("resolved", sa.Integer(), nullable=False), + sa.Column("total", sa.Integer(), nullable=False), + sa.Column( + "config", + sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql"), + server_default=sa.text("'{}'"), + nullable=False, + ), + sa.Column( + "recorded_by", + sa.Uuid(as_uuid=True), + sa.ForeignKey("app_user.id", ondelete="SET NULL"), + nullable=True, + ), + sa.UniqueConstraint("workspace_id", "benchmark_suite_id", name=_UNIQUE_NAME), + ) + op.create_index(_INDEX_NAME, _TABLE, ["workspace_id"]) + + +def downgrade() -> None: + if _TABLE not in _existing_tables(): + return + indexes = {ix["name"] for ix in sa.inspect(op.get_bind()).get_indexes(_TABLE)} + if _INDEX_NAME in indexes: + op.drop_index(_INDEX_NAME, table_name=_TABLE) + op.drop_table(_TABLE) diff --git a/packages/db/tests/test_models.py b/packages/db/tests/test_models.py index d793141e..94f1b0cd 100644 --- a/packages/db/tests/test_models.py +++ b/packages/db/tests/test_models.py @@ -122,6 +122,8 @@ # F35 benchmark-leaderboard tables. "BenchmarkSuite", "BenchmarkSubmission", + # F41 Self-Eval Gate baseline. + "SelfEvalBaseline", # F36 human-approval-system tables. "ApprovalDecision", "PolicyOverrideGrant",