diff --git a/.github/workflows/fred.yml b/.github/workflows/fred.yml index 0f860bd..0102aef 100644 --- a/.github/workflows/fred.yml +++ b/.github/workflows/fred.yml @@ -42,3 +42,100 @@ jobs: # in-process over an injected httpx transport. - name: Coverage (unit + integration) run: make coverage + + # Scores every task's real verifier against its oracle, an empty run, three + # bypass routes, and a delegated call. No model involved. + - name: Validate eval task verifiers + run: make validate-tasks + + evals: + # THE MERGE GATE on the tool surface: drives every task with the real claude-code + # agent against the mock FRED API, and fails unless the rewards clear the threshold. + # + # `check` above runs make validate-tasks, which is a different question: it proves + # each verifier accepts its oracle and rejects a bypass, with no model involved. + # That catches a broken verifier. Only this catches a server a real agent cannot + # actually drive. + # + # Unlike every other step here this one spends model tokens, so it needs `check` to + # pass first and is guarded to same-repo PRs (a fork PR gets no secrets). + # + # It spends them against a Claude subscription rather than API credits. + # CLAUDE_FORCE_OAUTH makes harbor's claude-code agent blank ANTHROPIC_API_KEY before + # building the container env, so only CLAUDE_CODE_OAUTH_TOKEN survives and the CLI + # falls back to subscription auth. Never add ANTHROPIC_API_KEY back: with both + # present and the flag unset the CLI silently prefers the key and the run is on + # credits again, with only a debug log to say so. + # + # Serialized on purpose. Subscription rate limits are per account and shared with + # interactive use. + runs-on: ubuntu-latest + needs: [check] + if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }} + concurrency: + group: fred-evals + cancel-in-progress: false + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_FORCE_OAUTH: "1" + # For --upload. Results land on the hub as `ci-evals-fred`, following the repo-wide + # `ci-evals-` convention so every plugin's CI history is searchable together. + HARBOR_API_KEY: ${{ secrets.HARBOR_API_KEY }} + # One attempt per task. A task that only passes sometimes is a signal to fix the + # task, not to average it away. + EVAL_ATTEMPTS: "1" + EVAL_MIN_MEAN: "1.0" + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.13" + + # A setup-token credential is long-lived but not permanent. Without this probe an + # expired one surfaces as "the gate did not clear 1.0", which reads as a server + # regression and isn't. Fails rather than skips on a missing token: a + # secret-dependent job silently going green is the worse failure. + - name: Check the subscription token + run: | + set -euo pipefail + if [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then + echo "::error::CLAUDE_CODE_OAUTH_TOKEN is not set. Mint one with 'claude setup-token' and add it as a repository secret." + exit 1 + fi + status="$(curl -sS -o /tmp/auth-probe.json -w '%{http_code}' \ + https://api.anthropic.com/v1/messages \ + -H "Authorization: Bearer $CLAUDE_CODE_OAUTH_TOKEN" \ + -H "anthropic-version: 2023-06-01" \ + -H "anthropic-beta: oauth-2025-04-20" \ + -H "content-type: application/json" \ + -d '{"model":"claude-haiku-4-5","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}')" + if [ "$status" = "401" ]; then + echo "::error::CLAUDE_CODE_OAUTH_TOKEN is expired or revoked. Mint a new one with 'claude setup-token' and update the repository secret." + exit 1 + fi + if [ "$status" != "200" ]; then + echo "::error::Auth probe returned HTTP $status, expected 200. The gate would likely fail after spending a rate-limit window; response body:" + cat /tmp/auth-probe.json + exit 1 + fi + echo "Auth probe returned HTTP 200; proceeding." + + - name: Run the eval gate (claude-code) + env: + EVALS_OUT_DIR: ${{ runner.temp }}/eval-trials + # Every gate run uploads, PR included: gating uploads to main left a PR gate + # with nothing on the hub, so the only record was an artifact expiring in 7 days. + EVALS_UPLOAD: "1" + run: make evals + + # always(), not failure(): harbor draws progress as a live TUI, so a run that + # passes slowly logs almost nothing. The trials are the only record a PR run + # produces, and a failure is only diagnosable from the trajectory. + - name: Upload trials for diagnosis + if: always() + uses: actions/upload-artifact@v4 + with: + name: fred-eval-trials + path: ${{ runner.temp }}/eval-trials + retention-days: 7 diff --git a/plugins/fred/.claude-plugin/plugin.json b/plugins/fred/.claude-plugin/plugin.json index 2bfc229..c0f2ba2 100644 --- a/plugins/fred/.claude-plugin/plugin.json +++ b/plugins/fred/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fred", - "version": "0.5.0", + "version": "0.6.0", "description": "MCP server for the FRED API: economic time series from the St. Louis Fed, with search, aligned multi-series observations, revision history, and the release calendar.", "author": { "name": "Walker Hughes" diff --git a/plugins/fred/.gitignore b/plugins/fred/.gitignore index 29d2eea..d266e1e 100644 --- a/plugins/fred/.gitignore +++ b/plugins/fred/.gitignore @@ -14,3 +14,6 @@ wheels/ # Coverage .coverage + +# Harbor job output +evals/jobs/ diff --git a/plugins/fred/.mcp.json b/plugins/fred/.mcp.json index 1882156..e1e5337 100644 --- a/plugins/fred/.mcp.json +++ b/plugins/fred/.mcp.json @@ -6,6 +6,7 @@ "args": ["${CLAUDE_PLUGIN_ROOT}/scripts/start-server.sh"], "env": { "FRED_API_KEY": "${FRED_API_KEY:-}", + "FRED_BASE_URL": "${FRED_BASE_URL:-}", "FRED_LOG_LEVEL": "${FRED_LOG_LEVEL:-}" } } diff --git a/plugins/fred/Makefile b/plugins/fred/Makefile index 702216a..bd613ef 100644 --- a/plugins/fred/Makefile +++ b/plugins/fred/Makefile @@ -1,4 +1,5 @@ -.PHONY: lint lint-fix format typecheck check test test-unit test-integration coverage +.PHONY: lint lint-fix format typecheck check test test-unit test-integration coverage \ + validate-tasks evals mock-api benchmark-build benchmark benchmark-view lint: uv run ruff check . @@ -25,3 +26,37 @@ test-integration: coverage: uv run pytest --cov --cov-report=term-missing + +# Check every eval task verifier against its oracle and against each bypass route. No +# model and no API key, so it runs in CI. Needs Docker: rewardkit scores these checks +# and does not build on macOS. +validate-tasks: + uv run python evals/generate_tasks.py + bash evals/validate_local.sh + python3 evals/check_reward.py --selftest + +# THE MERGE GATE: drives the tasks with a real agent. Spends model tokens, needs Docker +# and CLAUDE_CODE_OAUTH_TOKEN. validate-tasks proves the verifiers work; only this +# proves an agent can actually drive the server. +evals: + bash evals/run_gate.sh + +# Run the mock FRED API on its own, the way the eval benchmark runs it. +mock-api: + uv run python -m tests.fixtures.fred_api 8080 + +# Benchmark targets (need Docker running and CLAUDE_CODE_OAUTH_TOKEN set). These cd into +# evals/ so Harbor resolves the task path correctly regardless of where you invoke make, +# and call Harbor through uv so it does not need to be on PATH. Pinned to the version +# the tasks were validated against. +HARBOR ?= uv tool run --from "harbor==0.18.0" harbor + +# Context is the plugin root so the image can copy the working tree it is testing. +benchmark-build: + docker build -f evals/environment/Dockerfile -t fred-bench . + +benchmark: + cd evals && $(HARBOR) run -c job.yaml + +benchmark-view: + cd evals && $(HARBOR) view jobs diff --git a/plugins/fred/README.md b/plugins/fred/README.md index c50ffa5..f82d027 100644 --- a/plugins/fred/README.md +++ b/plugins/fred/README.md @@ -92,16 +92,21 @@ What economic data just came out, and what is scheduled next, split around today ```bash uv sync -make check # lint, typecheck, unit tests -make test # everything -make coverage # with a report, 80% floor +make check # lint, typecheck, unit tests +make test # everything +make coverage # with a report, 80% floor +make validate-tasks # score every eval verifier against its oracle and each bypass (needs Docker) +make evals # THE MERGE GATE: drive the tasks with a real agent (needs Docker + a token) ``` Integration tests drive the registered MCP server against a mock FRED built from trimmed real captures. No network and no API key, so the whole suite runs anywhere. +The agent-loop benchmark lives in [`evals/`](evals/): 10 tasks over all five tools, each scoring both whether the answer is right and whether it came through the MCP server rather than round it. See [evals/README.md](evals/README.md). + ## Not here - **Maps / GeoFRED.** A different product with a different shape. - **Tag and category tree browsing.** `search_series` covers the reachable ground; the tree is a UI affordance. - **Sources.** Metadata about metadata. - **A response cache.** FRED allows 120 requests a minute and the data moves slowly, so nothing is under pressure. See the design doc's deferred work. +- **Forward date spans.** `"5y"` means five years ago; a forward window needs an absolute end date. diff --git a/plugins/fred/docs/design.md b/plugins/fred/docs/design.md index 5eee910..a9384f6 100644 --- a/plugins/fred/docs/design.md +++ b/plugins/fred/docs/design.md @@ -164,16 +164,36 @@ leave a model holding three IDs with no idea which to fix. ## Evals -Unit-level misuse tests (`tests/unit/test_schemas.py`, `test_dates.py`) feed realistic model +Three layers, answering three different questions. + +**Unit-level misuse tests** (`tests/unit/test_schemas.py`, `test_dates.py`) feed realistic model mistakes through correction and validation and assert the corrections and the suggestion-bearing -errors. Integration tests drive the registered MCP server against a mock FRED built from trimmed -real captures, so the shaping is asserted against FRED's shapes rather than invented ones. All of -it runs in CI with no key and no network. +errors. + +**Integration tests** drive the registered MCP server against a mock FRED built from trimmed real +captures, so the shaping is asserted against FRED's shapes rather than invented ones. + +**The Harbor benchmark** (`evals/`) drives Claude Code over 10 tasks covering all five tools and +fails unless every reward is 1.0. This is the only layer that can catch a server an agent cannot +drive, which is a different failure from a server that computes the wrong thing. Every task scores +two rewards: `outcome` (the answer is right) and `process` (it came through `mcp__fred__*`, and +nothing went round the server). + +The split exists because a tastytrade gate run produced a perfect answer without calling a single +tool: it read the mock's source off disk and drove the backend directly. This plugin has three such +routes rather than two, since the benchmark runs with the network up and `api.stlouisfed.org` is +therefore reachable, so `process` matches on the mock's port, the fixture module name, and the real +API's hostname. `require-local-api` is the hard stop behind that: the server refuses to start +unless `FRED_BASE_URL` points at localhost, so a benchmark run cannot spend a real key. + +One task is worth naming. `rate-history-max` asks for the highest value a long daily series ever +reached, and the fixture puts that maximum on a single day at an index the downsampler does not +sample: the summary says 300.0 and the best point actually returned is 199.96, against a tolerance +of 0.01. It is the summary-before-downsampling claim above, turned into pass or fail. -There is no Harbor agent-loop benchmark, unlike the tastytrade server. That one exists because -order placement makes a wrong answer expensive; FRED is read-only. A benchmark is worth adding once -the tool surface has settled, and it would measure the table above at the agent loop rather than at -the payload. +All three layers use the same fixtures through the same `route()` function, so the benchmark and +the test suite cannot disagree about what FRED returns. The first two run in CI on every PR with no +key and no network; `evals/README.md` covers the third. ## Deferred work @@ -184,3 +204,6 @@ the payload. - **Maps/GeoFRED.** Regional data by shape rather than by series. - **Paging.** Every tool returns a bounded page with FRED's own total. No cursor, because no question so far has needed the second page. +- **Forward date spans.** `start="5y"` means five years *ago*; there is no spelling for "the next + 60 days", so a forward window is written out as an absolute date. The calendar's default window + already straddles today, which is what the common question needs. diff --git a/plugins/fred/evals/README.md b/plugins/fred/evals/README.md new file mode 100644 index 0000000..6c5313e --- /dev/null +++ b/plugins/fred/evals/README.md @@ -0,0 +1,233 @@ +# Evals + +The server is evaluated at the agent-loop level, where it actually runs: Claude Code +drives the tools over a set of tasks using the +[Harbor](https://github.com/laude-institute/harbor) framework, and each trial records +reward, tool calls, tokens, and latency. + +Every task runs against the mock FRED API in `tests/fixtures/fred_api.py`, so the answers +are fixed and reproducible and no run touches the real API or spends a real key. That is +not only a safety property: FRED publishes new observations every month and revises old +ones, so a gate scored against live data would fail on the day CPI comes out. + +The fast deterministic checks for argument correction and error guidance live alongside +the unit tests, in `tests/unit/test_schemas.py` and `tests/unit/test_dates.py`. + +## Layout + +``` +evals/ + environment/ + Dockerfile # python, uv, the server checkout, rewardkit + scripts/ # require-local-api, start-mock, mcp-server + tasks// + task.toml # task config + instruction.md # the prompt the agent sees + tests/test.sh # verifier: `rewardkit /tests` + tests/outcome/check.py # reward 1: the answer is right + tests/process/check.py # reward 2: it came through the MCP server + solution/solve.sh # oracle, writes the known-correct answer + job.yaml # runs the agent over every task + generate_tasks.py # regenerates the tasks from the fixtures + check_reward.py # gates a harbor result.json on its rewards + explain_trials.py # names each trial and dumps the tool calls behind a failure + validate_local.sh # scores every verifier without Harbor or a model + validate_in_container.sh # the reward matrix it asserts +``` + +## Tasks (10) + +One per meaningful capability, covering all five tools. + +| Task | Tool | Asks for | +|---|---|---| +| `unemployment-latest` | `get_observations` | the latest UNRATE value | +| `inflation-yoy` | `get_observations` | CPI year-over-year, which means `units=pc1` | +| `rate-history-max` | `get_observations` | the highest value a long daily series ever reached | +| `gdp-peak-unemployment` | `get_observations` | UNRATE on the date GDPC1 peaked, so two series on one index | +| `initial-print` | `get_revisions` | GDP as first published, before revision | +| `revision-count` | `get_revisions` | how many observations have been revised | +| `next-release` | `get_release_calendar` | the next scheduled Employment Situation date | +| `release-series-count` | `search_series` | how many monthly series a release publishes | +| `series-units` | `get_series` | the units of a series, as text | +| `find-series-id` | `search_series` | the canonical ID for a described series | + +**`rate-history-max` is the one that earns its keep.** The daily fixture puts its true +maximum on a single day at an index the downsampler does not sample, so: + +``` +summary max 300.0 +highest point actually returned 199.96 +``` + +An agent that reads the returned points is wrong by a hundred, and the tolerance is 0.01. +The task is the central claim of `get_observations`, that the summary covers every +observation while the points are only a sample, turned into pass or fail. +`tests/integration/test_observations.py::test_the_extremes_are_not_in_the_returned_points` +fails if a fixture change ever makes the peak reachable from the points again, which is +how an earlier version of the fixture quietly made this task prove nothing. + +Nothing in the tasks is hand-typed. `generate_tasks.py` computes every expected answer +from the fixtures by running the same shaping code the server uses, so a task can never +disagree with the data the agent sees. Regenerate after changing a fixture: + +```bash +python evals/generate_tasks.py +``` + +`next-release` is the exception to "fixed answers", and deliberately. The fixture places +release dates relative to the current day, because a calendar whose whole job is "what +came out and what is next" stops straddling today the moment a hard-coded date ages. Its +check and its oracle both compute the expected date at run time, and the check accepts +today's and yesterday's answer, since a trial that starts before midnight is graded after +it. + +## Two rewards per task + +Every task scores `outcome` and `process`, both computed by +[rewardkit](https://pypi.org/project/harbor-rewardkit/) from the subdirectories of +`tests/`. + +`outcome` is the answer. `process` is whether it came through the plugin. + +The split is not theoretical. A tastytrade gate run produced a perfect answer without +ever calling a tool: it searched for the MCP tools, never called them, read the mock's +source off disk, and drove the backend with `urllib`. `outcome` alone scored that 1.0. + +This plugin has **three** ways round the server rather than tastytrade's two, and +`process` has to see all of them: + +| Route | Why it is reachable | How it is caught | +|---|---|---| +| The local mock's port | the server needs a backend | matched on `:8080`, not on a hostname. The mock binds every interface, so it answers on `localhost`, `127.0.0.1`, `0.0.0.0`, `[::1]` and the container's own name; naming two of those lets the other three through | +| The fixtures on disk | they hold every expected answer in plain Python | matched on the module name | +| The real FRED API | `network_mode: public` is required for the agent to reach its own model | matched on the API hostname | + +The third is new relative to tastytrade, whose mock-only setup never had to consider it. +It is also why `require-local-api` exists: the server refuses to start unless +`FRED_BASE_URL` points at localhost, so a benchmark run cannot spend a developer's real +key even if one leaked into the environment. + +A delegated call counts, and seeing it takes a second source. `trajectory.json` holds the +main session only: harbor's session scan drops any jsonl whose path contains a +`subagents/` component, which is exactly where Claude Code writes a subagent's transcript. +So the checks read the raw session transcripts under `/logs/agent/sessions` as well, and +stop caring who placed the call. Whether the top-level agent called the tool or routed it +through a delegate is the harness's decision, not a fact about this plugin. + +That has to cut both ways. Crediting a delegated MCP call while missing a delegated `curl` +would turn "ask a subagent" into an invisible bypass, so the bypass criterion reads the +same union, and the reward matrix asserts both directions. + +Both checks fail closed. No trajectory means no evidence the intended route was taken, so +`process` is 0. That is why the oracle scores `outcome=1, process=0`: it is a shell +script, not an agent, and cannot call tools. + +Every task prompt states the rule `process` scores, including the paragraph explaining +that an MCP tool with a deferred schema is loaded with `ToolSearch` and then called +directly. That paragraph is inherited from tastytrade, where four gate trials failed the +same way: each opened with `ToolSearch`, loaded a schema, and then never called the tool, +one of them running `Bash: mcp__tastytrade__get_option_chain ...` as a shell command until +it timed out. The confusion was never about which tool or which arguments; it was about +how to invoke an MCP tool at all. + +The wording avoids the port, the fixture module name, and the API hostname the bypass +check greps for, so that an agent echoing its instructions into a shell comment cannot +fail the check by quoting it. + +## Check the verifiers without Harbor + +`validate_local.sh` scores every task's real verifier against seven synthetic +trajectories and asserts the whole reward matrix. No model, no Harbor, no API key: + +| case | answer | trajectory | outcome | process | +|---|---|---|---|---| +| solved | oracle | called an MCP tool | 1 | 1 | +| empty | none | none | 0 | 0 | +| bypassed-port | oracle | curled the local mock | 1 | 0 | +| bypassed-fixture | oracle | read the fixtures off disk | 1 | 0 | +| bypassed-real | oracle | curled the real FRED API | 1 | 0 | +| delegated | oracle | subagent called an MCP tool | 1 | 1 | +| delegated-bypass | oracle | subagent curled the local mock | 1 | 0 | + +The bypass rows are the point, and they are what `harbor run -a oracle` cannot tell you. +One spelling of a bypass proves only that one spelling is caught, so `bypassed-port` +deliberately uses `0.0.0.0`, the address a hostname list would miss. + +```bash +make validate-tasks +# 70 passed, 0 failed +``` + +It runs in the bench image rather than on the host, because rewardkit scores these checks +and does not build on macOS (its litellm dependency wants a newer rustc than ships there). +Using the same image CI uses also means the verifier under test is the one that will +really grade a gate run, so **this needs Docker**. + +## Run it + +Harbor must run with `evals/` as the working directory, because it resolves the dataset +path relative to where it is invoked. The `make` targets handle that, so run them from the +plugin root: + +```bash +export CLAUDE_CODE_OAUTH_TOKEN=... # claude setup-token +make benchmark-build # docker build -f evals/environment/Dockerfile -t fred-bench . +make benchmark # cd evals && harbor run -c job.yaml +make benchmark-view # cd evals && harbor view jobs +``` + +Each task carries a one-line `environment/Dockerfile` (`FROM fred-bench`). Harbor only +discovers a directory as a task if it has an `environment/`, so this is required even +though the task also sets `docker_image`. + +The image copies the working tree rather than cloning a ref, so a run measures the code +you have checked out. Rebuild after changing the server. + +## The merge gate + +`make validate-tasks` and `make evals` answer different questions, and CI runs both. +`validate-tasks` proves each verifier accepts its oracle and rejects every bypass, with no +model involved, which catches a broken verifier. `make evals` drives all 10 tasks with the +real claude-code agent and fails unless every reward is 1.0, which is the only thing that +catches a server an agent cannot actually drive. + +The gate authenticates with `CLAUDE_CODE_OAUTH_TOKEN` so runs bill to a Claude +subscription rather than API credits. It deliberately does **not** accept +`ANTHROPIC_API_KEY`: with a key present the CLI prefers it over the token, which either +moves the run onto credits silently or, if the key is empty, 401s every trial before +spending a token. + +Every gate run uploads to the Harbor hub as **`ci-evals-fred`**, one job per CI run +holding all 10 tasks as trials, following the repo-wide `ci-evals-` convention. + +```bash +export CLAUDE_CODE_OAUTH_TOKEN=... # claude setup-token +make evals # add HARBOR_API_KEY and EVALS_UPLOAD=1 to upload +``` + +### When it fails + +The gate prints each trial by name with its rewards, and for any trial that lost +`process`, the tool calls the agent made, MCP ones unmarked and everything else flagged +`!`. That list *is* the `process` score, so it is usually the whole diagnosis: + +``` +== inflation-yoy: outcome=1.0, process=0.5 + 3 tool call(s), 1 through the MCP server: + ! Bash {"command": "curl -s http://0.0.0.0:8080/fred/series/observations?series_id=CPIAUCSL"} + mcp__fred__get_series {"series_ids": ["CPIAUCSL"]} +``` + +## Safety + +The agent never sees a real credential. The image sets `FRED_BASE_URL` to the local mock +and a throwaway key, and `require-local-api` refuses to start the server unless +`FRED_BASE_URL` points at localhost. The API is read-only in any case, so there is nothing +a benchmark run could change even if it did reach the real service. + +The mock runs in the container as a background process that the server wrapper starts on +first use, so the benchmark is not tied to the local Docker provider the way a +multi-container setup would be. It is a stdlib HTTP server over the same `route()` +function the unit tests drive through an httpx transport, so the benchmark and the test +suite cannot disagree about what FRED returns. diff --git a/plugins/fred/evals/check_reward.py b/plugins/fred/evals/check_reward.py new file mode 100644 index 0000000..7eaf4ac --- /dev/null +++ b/plugins/fred/evals/check_reward.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Gate a harbor ``result.json`` on its rewards. + +``harbor run`` exits 0 whatever the reward, so the gate has to inspect the job +result itself. Every fred task reports one reward in [0, 1], so the mean +over a task is its pass rate across attempts. + + python3 check_reward.py # every reward 1.0 + python3 check_reward.py --min-mean 0.9 # allow some slack + python3 check_reward.py --only outcome # one reward only + python3 check_reward.py --selftest + +Each task reports two rewards, ``outcome`` and ``process`` (see evals/README.md). +``--only`` restricts the gate to one of them, which is how the local validator +holds the oracle to ``outcome``: the oracle is a shell script, not an agent, so +it cannot call MCP tools and cannot score on ``process``. + +``--min-mean`` exists because this gate drives a real agent over 10 tasks, not +3. A single flaky task should not be indistinguishable from a broken server, +but the threshold is an explicit number in the workflow rather than a silent +default, so loosening it is a visible decision. +""" + +import json +import sys +from pathlib import Path + + +def rewards(stats: dict, only: str | None = None) -> tuple[list | None, str]: + """Every reward in the run, or (None, reason) if it did not complete cleanly. + + A task with one reward reports it under the metric name ({"mean": 1.0}); a + task with several reports them under the reward names ({"outcome": 1.0, + "process": 1.0}). `only` filters to one of those names. + """ + if stats.get("n_errored_trials") or not stats.get("n_completed_trials"): + return None, f"run did not complete cleanly (stats={stats})" + found = [ + value + for eval_stats in stats.get("evals", {}).values() + for metric in eval_stats.get("metrics", []) + for name, value in metric.items() + if only is None or name == only + ] + if not found: + return None, f"no {only or ''} rewards reported".replace(" ", " ") + return found, "" + + +def gate(stats: dict, min_mean: float = 1.0, only: str | None = None) -> tuple[bool, str]: + found, reason = rewards(stats, only) + if found is None: + return False, reason + mean = sum(found) / len(found) + failed = [r for r in found if r < min_mean] + n = stats["n_completed_trials"] + if min_mean >= 1.0 and failed: + return False, f"reward not perfect (mean={mean:.3f}, rewards={found})" + if mean < min_mean: + return False, f"mean reward {mean:.3f} below threshold {min_mean} (rewards={found})" + # `found` counts reward values, not tasks: harbor aggregates every trial into one + # mean per reward name, so a clean 10-task run reports two. Calling that "2 task(s)" + # reads like eight tasks went missing at exactly the moment someone is looking for + # a reason the gate failed. + return True, f"mean reward {mean:.3f} over {n} trial(s), {len(found)} reward(s)" + + +def _selftest() -> None: + ok = {"n_completed_trials": 2, "evals": {"a": {"metrics": [{"mean": 1.0}]}}} + assert gate(ok)[0] + assert not gate({**ok, "evals": {"a": {"metrics": [{"mean": 0.0}]}}})[0] + assert not gate({**ok, "n_errored_trials": 1})[0] + assert not gate({"n_completed_trials": 0, "evals": {}})[0] + assert not gate({"n_completed_trials": 1, "evals": {}})[0] + + mixed = { + "n_completed_trials": 2, + "evals": {"a": {"metrics": [{"mean": 1.0}]}, "b": {"metrics": [{"mean": 0.0}]}}, + } + assert not gate(mixed)[0], "one zero must fail a perfect gate" + assert not gate(mixed, min_mean=0.9)[0], "mean 0.5 is below 0.9" + assert gate(mixed, min_mean=0.5)[0], "mean 0.5 meets a 0.5 threshold" + + # Two named rewards per task. The oracle solves the answer but cannot call + # MCP tools, so it is outcome=1, process=0 and only `--only outcome` passes. + split = { + "n_completed_trials": 1, + "evals": {"a": {"metrics": [{"outcome": 1.0, "process": 0.0}]}}, + } + assert not gate(split)[0], "a zero process reward must fail the full gate" + assert gate(split, only="outcome")[0] + assert not gate(split, only="process")[0] + assert rewards(split, only="nope")[0] is None, "unknown reward name -> None" + print("check_reward selftest ok") + + +def main(argv: list[str]) -> int: + if argv[1:2] == ["--selftest"]: + _selftest() + return 0 + result_path, name = argv[1], argv[2] + min_mean = 1.0 + if "--min-mean" in argv: + min_mean = float(argv[argv.index("--min-mean") + 1]) + try: + stats = json.loads(Path(result_path).read_text()).get("stats", {}) + except FileNotFoundError: + print(f"{name}: no result.json at {result_path}", file=sys.stderr) + return 1 + only = argv[argv.index("--only") + 1] if "--only" in argv else None + ok, msg = gate(stats, min_mean, only) + print(f"{name}: {msg}", file=sys.stdout if ok else sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/plugins/fred/evals/environment/Dockerfile b/plugins/fred/evals/environment/Dockerfile new file mode 100644 index 0000000..3cd0c1a --- /dev/null +++ b/plugins/fred/evals/environment/Dockerfile @@ -0,0 +1,37 @@ +# Benchmark agent environment. +# +# One checkout of the server plus the mock FRED API, so Claude Code can be +# benchmarked over the eval tasks. Harbor installs the claude-code agent itself. +# +# Build context is the plugin root, not this directory: +# make benchmark-build # docker build -f evals/environment/Dockerfile . +FROM python:3.13-slim + +RUN apt-get update && apt-get install -y --no-install-recommends git curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# uv for dependency management (matches the project toolchain). +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +# The server under test is the working tree rather than a cloned ref, so a run +# measures the code actually in front of you rather than what has been pushed. +COPY . /opt/fred +RUN cd /opt/fred && (uv sync --frozen || uv sync) + +# rewardkit scores the verifiers (each tests// becomes a named reward). +# Baked in rather than fetched at verify time so verification needs no network, +# and in its own venv so it cannot disturb the server's resolved dependencies. +RUN python -m venv /opt/rewardkit \ + && /opt/rewardkit/bin/pip install --no-cache-dir "harbor-rewardkit==0.1.*" \ + && ln -s /opt/rewardkit/bin/rewardkit /usr/local/bin/rewardkit + +COPY evals/environment/scripts/ /usr/local/bin/ +RUN chmod +x /usr/local/bin/require-local-api /usr/local/bin/start-mock /usr/local/bin/mcp-server + +WORKDIR /app +# The server talks to the mock with a throwaway key. require-local-api also enforces the +# localhost target at startup, so a benchmark run can never reach the real FRED API or +# spend a real key: the container has no real key to spend and no route it would accept. +ENV FRED_BASE_URL=http://localhost:8080/fred \ + FRED_API_KEY=abcdef0123456789abcdef0123456789 \ + FRED_LOG_LEVEL=INFO diff --git a/plugins/fred/evals/environment/scripts/mcp-server b/plugins/fred/evals/environment/scripts/mcp-server new file mode 100644 index 0000000..3d349e0 --- /dev/null +++ b/plugins/fred/evals/environment/scripts/mcp-server @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Launch the server over stdio against the mock API. +# +# cd rather than `uv run --project`: the image's WORKDIR is /app so the agent writes its +# answer there, and `src` is only importable from the project root. --project alone sets +# the environment without moving the working directory. +set -euo pipefail +require-local-api +start-mock +cd /opt/fred +exec uv run python -m src.server diff --git a/plugins/fred/evals/environment/scripts/require-local-api b/plugins/fred/evals/environment/scripts/require-local-api new file mode 100644 index 0000000..e7a87f5 --- /dev/null +++ b/plugins/fred/evals/environment/scripts/require-local-api @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Refuse to run unless the server points at the local mock. This is the hard stop that +# keeps the benchmark from ever reaching the real FRED API, whatever the environment +# happens to say, and so from ever spending a developer's real key on a gate run. +set -euo pipefail +case "${FRED_BASE_URL:-}" in + http://localhost:* | http://127.0.0.1:*) exit 0 ;; +esac +echo "refusing to start: FRED_BASE_URL must point at the local mock (got '${FRED_BASE_URL:-unset}')." >&2 +exit 1 diff --git a/plugins/fred/evals/environment/scripts/start-mock b/plugins/fred/evals/environment/scripts/start-mock new file mode 100644 index 0000000..f7a4494 --- /dev/null +++ b/plugins/fred/evals/environment/scripts/start-mock @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Start the mock FRED API on :8080 if it isn't already running. Safe to call more than +# once, since only the first call actually launches it. +set -euo pipefail + +probe() { curl -sf "http://localhost:8080/fred/series?series_id=UNRATE" > /dev/null 2>&1; } + +probe && exit 0 + +cd /opt/fred +nohup uv run python -m tests.fixtures.fred_api 8080 > /tmp/mock-fred.log 2>&1 & + +for _ in $(seq 1 50); do + probe && exit 0 + sleep 0.2 +done +echo "mock FRED API failed to start" >&2 +cat /tmp/mock-fred.log >&2 +exit 1 diff --git a/plugins/fred/evals/explain_trials.py b/plugins/fred/evals/explain_trials.py new file mode 100644 index 0000000..e861844 --- /dev/null +++ b/plugins/fred/evals/explain_trials.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Say which trial scored what, and what the agent actually did. + +The gate used to dump every trial's verifier output with `cat /*/verifier/ +test-stdout.txt`, which prints thirteen anonymous pairs of numbers: + + outcome: 1.0 + process: 0.5 + +That says six tasks lost `process` and nothing about which six, let alone why. +Recovering it meant downloading the CI artifact, which needs credentials the +person reading the log may not have and which expires after seven days. + +So this prints the trial name with its rewards, and for any trial that did not +score a perfect `process`, the tool calls it made. `process` is entirely a +function of that list -- whether an `mcp__fred__*` call is in it, and +whether anything else went round the server -- so the list is the diagnosis. + + python3 explain_trials.py + +Deliberately does not re-implement the bypass regex. Duplicating it here would +give the log a second opinion that could drift from the checks, and the raw +calls are what a reader needs anyway. +""" + +import json +import sys +from pathlib import Path + +MCP_PREFIX = "mcp__fred__" +ARG_WIDTH = 160 + + +def _rewards(trial: Path) -> dict[str, float]: + """The trial's rewards, read from whatever the verifier left behind. + + rewardkit writes reward.json; its stdout carries the same numbers as + `name: value` lines. Harbor's layout for these has moved before, so both are + tried rather than pinning one path. + """ + for path in sorted(trial.rglob("reward.json")): + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + found = {k: float(v) for k, v in data.items() if isinstance(v, int | float)} + if found: + return found + + found = {} + for path in sorted(trial.rglob("test-stdout.txt")): + try: + text = path.read_text() + except OSError: + continue + for line in text.splitlines(): + name, _, value = line.partition(":") + try: + found[name.strip()] = float(value) + except ValueError: + continue + return found + + +def _calls(trial: Path) -> list[tuple[str, str]]: + """(tool name, arguments) for every call in the trial's trajectory.""" + for path in sorted(trial.rglob("trajectory.json")): + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + steps = data.get("steps") or [] + return [ + (str(call.get("function_name") or "?"), json.dumps(call.get("arguments") or {})) + for step in steps + for call in step.get("tool_calls") or [] + ] + return [] + + +def explain(job_dir: Path) -> None: + trials = sorted(p for p in job_dir.iterdir() if p.is_dir()) + if not trials: + print(f"no trial directories under {job_dir}") + return + + for trial in trials: + rewards = _rewards(trial) + if not rewards: + continue + summary = ", ".join(f"{name}={value}" for name, value in sorted(rewards.items())) + print(f"\n== {trial.name}: {summary}") + + if rewards.get("process", 0.0) >= 1.0: + continue + + calls = _calls(trial) + if not calls: + print(" no trajectory recorded, so `process` fails closed at 0") + continue + mcp = sum(1 for name, _ in calls if name.startswith(MCP_PREFIX)) + print(f" {len(calls)} tool call(s), {mcp} through the MCP server:") + for name, args in calls: + marker = " " if name.startswith(MCP_PREFIX) else "! " + print(f" {marker}{name} {args[:ARG_WIDTH]}") + + +if __name__ == "__main__": + explain(Path(sys.argv[1])) diff --git a/plugins/fred/evals/generate_tasks.py b/plugins/fred/evals/generate_tasks.py new file mode 100644 index 0000000..6ef671d --- /dev/null +++ b/plugins/fred/evals/generate_tasks.py @@ -0,0 +1,601 @@ +#!/usr/bin/env python3 +"""Generate the Harbor benchmark tasks. + +Every expected answer is computed here from the mock fixtures by running the same +shaping code the server uses. Nothing is typed in by hand, so a task answer can never +drift away from the data the agent actually sees. Change a fixture in +`tests/fixtures/fred_api.py` and rerun this script to refresh the tasks. + +Run: python evals/generate_tasks.py +""" + +import json +import os +import stat +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from src.shaping import align, parse_value, revision_rows, summarize # noqa: E402 +from tests.fixtures import fred_api as fx # noqa: E402 + +TASKS_DIR = os.path.join(os.path.dirname(__file__), "tasks") + + +# --- ground truth, computed from the fixtures ---------------------------------------- + + +def _pairs(rows): + return [(stamp, parse_value(value)) for stamp, value in rows] + + +def _latest_unrate(): + return summarize(_pairs(fx.UNRATE_OBS))["latest"] + + +def _cpi_yoy(): + return summarize(_pairs(fx.transform(fx.CPI_OBS, "pc1")))["latest"] + + +def _daily_max(): + return summarize(_pairs(fx.DAILY_OBS))["max"] + + +def _unemployment_at_gdp_peak(): + """Needs both series on one date index, which is what the tool does for you.""" + dates, columns = align({"UNRATE": _pairs(fx.UNRATE_OBS), "GDPC1": _pairs(fx.GDPC1_OBS)}) + gdp = columns["GDPC1"] + peak = max(range(len(dates)), key=lambda i: (gdp[i] is not None, gdp[i])) + return columns["UNRATE"][peak] + + +def _initial_print(): + return dict(_pairs(fx.INITIAL_OBS["GDPC1"]))["2025-01-01"] + + +def _revised_count(): + rows = revision_rows(_pairs(fx.INITIAL_OBS["GDPC1"]), _pairs(fx.GDPC1_OBS)) + return float(sum(1 for row in rows if row.get("revision"))) + + +def _monthly_series_in_release_50(): + monthly = [s for s in fx.RELEASE_50_SERIES if s["frequency"].lower() == "monthly"] + return float(len(monthly)) + + +def _gdp_units(): + return fx.GDPC1["units"] + + +# --- task definitions ---------------------------------------------------------------- + +# (name, prompt, answer key, value, tolerance) +NUMERIC_TASKS = [ + ( + "unemployment-latest", + "What is the most recent US unemployment rate (series UNRATE), as a percent?", + "unemployment_rate", + _latest_unrate(), + 0.01, + ), + ( + "inflation-yoy", + "For the CPI series CPIAUCSL, find the most recent year-over-year change, as a percent. " + "The server can compute that transformation for you; do not do the arithmetic by hand.", + "cpi_yoy_pct", + _cpi_yoy(), + 0.01, + ), + ( + "rate-history-max", + "For the 10-year Treasury series DGS10, find the highest value it has ever reached over " + "its entire history in this dataset. Be careful: a long daily series is returned as a " + "sample of its points, and the highest point is not necessarily among the ones you get " + "back. The tool reports the true figure alongside the points.", + "max_yield", + _daily_max(), + 0.01, + ), + ( + "gdp-peak-unemployment", + "Across the observations available for real GDP (GDPC1), find the date on which it was " + "highest, then report the US unemployment rate (UNRATE) for that same date, as a percent.", + "unemployment_rate", + _unemployment_at_gdp_peak(), + 0.01, + ), + ( + "initial-print", + "Real GDP (GDPC1) for the observation dated 2025-01-01 has since been revised. Find the " + "value as it was FIRST published, not the value it holds today.", + "initial_value", + _initial_print(), + 0.5, + ), + ( + "revision-count", + "Across the observations available for real GDP (GDPC1), count how many have been revised " + "since they were first published, that is, how many now hold a different value than the " + "one first reported.", + "revised_count", + _revised_count(), + 0.01, + ), + ( + "release-series-count", + "The Employment Situation release has FRED release id 50. Count how many of the series it " + "publishes are monthly.", + "series_count", + _monthly_series_in_release_50(), + 0.01, + ), +] + +# (name, prompt, answer key, expected string) +STRING_TASKS = [ + ( + "series-units", + "What are the units of the real GDP series GDPC1? Report the full units description " + "exactly as FRED gives it.", + "units", + _gdp_units(), + ), + ( + "find-series-id", + "Find the FRED series ID for the headline US unemployment rate: the monthly, seasonally " + "adjusted one that is the most widely used series of its kind.", + "series_id", + "UNRATE", + ), +] + + +# --- prompt boilerplate -------------------------------------------------------------- +# +# Deliberately worded without the port, the fixture module name, or the API hostname that +# the bypass check greps for. An agent that echoes its instructions into a shell comment +# or a todo entry would otherwise fail the very check the paragraph exists to pass. +MCP_ROUTE = """\ +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server.""" + +TASK_TOML = """\ +[task] +name = "fred-mcp/{name}" +description = "{desc}" + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 +""" + +NUMERIC_INSTRUCTION = """\ +# Task: {title} + +{instruction} + +{route} + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{{"{key}": }} +``` +""" + +STRING_INSTRUCTION = """\ +# Task: {title} + +{instruction} + +{route} + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{{"{key}": ""}} +``` +""" + +# Every task's verifier is the same line: rewardkit scores each subdirectory of tests/ +# as its own named reward. The image installs it (evals/environment/Dockerfile). +TEST_SH = """\ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests +""" + +OUTCOME_NUMERIC = '''\ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "{key}" +EXPECTED = {expected!r} +TOLERANCE = {tol!r} + + +@criterion(description="answer.json[{key}] is within {tol} of {expected}") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False +''' + +OUTCOME_STRING = '''\ +"""`outcome` reward: the text in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. Compared case-insensitively with whitespace +collapsed, since "Billions of Chained 2017 Dollars" and "billions of chained 2017 +dollars" are the same answer and neither is more correct. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "{key}" +EXPECTED = {expected!r} + + +def _normal(value: object) -> str: + return " ".join(str(value).split()).strip().lower() + + +@criterion(description="answer.json[{key}] equals {expected}") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return _normal(data[KEY]) == _normal(EXPECTED) + except Exception: + return False +''' + +# The calendar answer is a date relative to the day the trial runs, because the fixture +# generates release dates from a clock rather than writing them down. A fixed date would +# be right for exactly one day. +OUTCOME_NEXT_RELEASE = '''\ +"""`outcome` reward: the next release date for release 50. + +Generated by evals/generate_tasks.py. The fixture places release dates relative to the +current day, so the expected answer is computed here rather than baked in: a literal +date would stop being the right answer tomorrow. +""" + +import json +from datetime import date, timedelta +from pathlib import Path + +from rewardkit import criterion + +KEY = "{key}" +OFFSET_DAYS = {offset} + + +def _accepted() -> set: + """The expected date, plus the one for yesterday. + + The agent and the verifier are two processes, and a trial that starts just before + midnight is graded on the next day against a fixture that has moved with it. + Accepting both is a tolerance on the clock, the same as the numeric tasks carry a + tolerance on the value. + """ + today = date.today() + return {{(day + timedelta(days=OFFSET_DAYS)).isoformat() for day in (today, today - timedelta(days=1))}} + + +@criterion(description="answer.json[{key}] is the next scheduled release date") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return str(data[KEY]).strip() in _accepted() + except Exception: + return False +''' + +PROCESS_MCP = '''\ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\\b|\\bfred_api\\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True +''' + +NUMERIC_SOLVE = """\ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${{APP_DIR:-/app}}" +mkdir -p "$APP_DIR" +echo '{{"{key}": {expected}}}' > "$APP_DIR/answer.json" +""" + +STRING_SOLVE = """\ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${{APP_DIR:-/app}}" +mkdir -p "$APP_DIR" +cat > "$APP_DIR/answer.json" <<'JSON' +{json_line} +JSON +""" + +NEXT_RELEASE_SOLVE = """\ +#!/usr/bin/env bash +# Oracle for the calendar task. The expected date moves with the clock, so the oracle +# computes it the same way the verifier does rather than echoing a literal. +set -euo pipefail +APP_DIR="${{APP_DIR:-/app}}" +mkdir -p "$APP_DIR" +python3 - "$APP_DIR/answer.json" <<'PY' +import json, sys +from datetime import date, timedelta +answer = (date.today() + timedelta(days={offset})).isoformat() +with open(sys.argv[1], "w") as fh: + json.dump({{"{key}": answer}}, fh) +PY +""" + +ENVIRONMENT_DOCKERFILE = "FROM fred-bench\n" + + +def _write(path: str, content: str, executable: bool = False) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + fh.write(content) + if executable: + os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def _title(name: str) -> str: + return name.replace("-", " ").title() + + +def _common(base: str, name: str, desc: str, instruction: str, template: str, key: str) -> None: + _write(os.path.join(base, "task.toml"), TASK_TOML.format(name=name, desc=desc.replace('"', "'"))) + _write( + os.path.join(base, "instruction.md"), + template.format(title=_title(name), instruction=instruction, key=key, route=MCP_ROUTE), + ) + _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) + _write(os.path.join(base, "tests", "process", "check.py"), PROCESS_MCP) + + +def generate() -> list[str]: + names: list[str] = [] + + for name, instruction, key, expected, tol in NUMERIC_TASKS: + base = os.path.join(TASKS_DIR, name) + _common(base, name, instruction, instruction, NUMERIC_INSTRUCTION, key) + _write( + os.path.join(base, "tests", "outcome", "check.py"), + OUTCOME_NUMERIC.format(key=key, expected=expected, tol=tol), + ) + _write( + os.path.join(base, "solution", "solve.sh"), + NUMERIC_SOLVE.format(key=key, expected=expected), + executable=True, + ) + names.append(name) + + for name, instruction, key, expected in STRING_TASKS: + base = os.path.join(TASKS_DIR, name) + _common(base, name, instruction, instruction, STRING_INSTRUCTION, key) + _write(os.path.join(base, "tests", "outcome", "check.py"), OUTCOME_STRING.format(key=key, expected=expected)) + _write( + os.path.join(base, "solution", "solve.sh"), + STRING_SOLVE.format(json_line=json.dumps({key: expected})), + executable=True, + ) + names.append(name) + + # The calendar task: a clock-relative answer, so both the check and the oracle + # compute it rather than carrying a literal. + name, key, offset = "next-release", "next_release_date", fx.NEXT_RELEASE_50_OFFSET + base = os.path.join(TASKS_DIR, name) + instruction = ( + "The Employment Situation release has FRED release id 50. Find the date of its NEXT " + "scheduled release, that is, the first one still in the future. Report it as YYYY-MM-DD." + ) + _common(base, name, instruction, instruction, STRING_INSTRUCTION, key) + _write( + os.path.join(base, "tests", "outcome", "check.py"), + OUTCOME_NEXT_RELEASE.format(key=key, offset=offset), + ) + _write( + os.path.join(base, "solution", "solve.sh"), + NEXT_RELEASE_SOLVE.format(key=key, offset=offset), + executable=True, + ) + names.append(name) + + # Every task needs an environment/ directory or Harbor will not discover it. + for task in names: + _write(os.path.join(TASKS_DIR, task, "environment", "Dockerfile"), ENVIRONMENT_DOCKERFILE) + + return names + + +if __name__ == "__main__": + created = generate() + print(f"Generated {len(created)} tasks:") + for task in created: + print(" -", task) diff --git a/plugins/fred/evals/job.yaml b/plugins/fred/evals/job.yaml new file mode 100644 index 0000000..b6799d0 --- /dev/null +++ b/plugins/fred/evals/job.yaml @@ -0,0 +1,33 @@ +# Run Claude Code against the server over every task. Harbor records the reward, phase +# timings, and token and cost totals per trial. Read them with `harbor view jobs`. +# +# Before running: +# make benchmark-build # from the plugin root +# export CLAUDE_CODE_OAUTH_TOKEN=... +# make benchmark +jobs_dir: jobs +n_attempts: 3 # trials per task + +orchestrator: + type: local + n_concurrent_trials: 4 + +environment: + type: docker +# No ANTHROPIC_API_KEY here on purpose. The agent authenticates with +# CLAUDE_CODE_OAUTH_TOKEN, which harbor's claude-code agent forwards when +# CLAUDE_FORCE_OAUTH is set. Declaring the key puts an empty ANTHROPIC_API_KEY in +# the container, the CLI prefers it over the token, and every trial dies on a 401 +# before spending a token. Adding it back on a machine that has one also silently +# moves the run onto API credits. + +agents: + - name: claude-code + model_name: anthropic/claude-haiku-4-5-20251001 + mcp_servers: + - name: fred + transport: stdio + command: mcp-server + +datasets: + - path: ./tasks diff --git a/plugins/fred/evals/run_gate.sh b/plugins/fred/evals/run_gate.sh new file mode 100755 index 0000000..bdcf36d --- /dev/null +++ b/plugins/fred/evals/run_gate.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Gate runner (backs `make evals`): drives the eval tasks with the claude-code +# agent against the mock FRED API, and fails unless the rewards clear a +# threshold. This is the merge gate on the tool surface. +# +# `make validate-tasks` is the other half and a different question: it checks +# that each verifier accepts its oracle and rejects an empty answer, with no +# model involved. That catches a broken verifier. Only this catches a server a +# real agent cannot drive. +# +# Hub results are named `ci-evals-fred`. That is the repo-wide convention, +# `ci-evals-`, so every plugin's CI runs are searchable together on the +# hub and no plugin's history hides behind a generic name. +# +# Required: +# CLAUDE_CODE_OAUTH_TOKEN subscription auth for the agent +# docker, running +# Optional: +# EVAL_ATTEMPTS trials per task (default 1; job.yaml uses 3 interactively) +# EVAL_MIN_MEAN reward threshold (default 1.0, every task must pass) +# EVALS_OUT_DIR keep the trials here instead of a temp dir +# EVALS_UPLOAD set to push results to the Harbor hub (needs HARBOR_API_KEY) +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HARBOR="${HARBOR:-uv tool run --from harbor==0.18.0 harbor}" +ATTEMPTS="${EVAL_ATTEMPTS:-1}" +MIN_MEAN="${EVAL_MIN_MEAN:-1.0}" +JOB_NAME="ci-evals-fred" + +die() { echo "error: $1" >&2; exit 1; } + +docker info > /dev/null 2>&1 || die "docker is not running" +# Subscription auth only. An ANTHROPIC_API_KEY in the environment would be +# preferred by the CLI over the token and quietly move the run onto credits, so +# it is not accepted as a fallback here. +[ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] \ + || die "CLAUDE_CODE_OAUTH_TOKEN is not set (mint one with: claude setup-token)" +if [ -n "${EVALS_UPLOAD:-}" ] && [ -z "${HARBOR_API_KEY:-}" ]; then + die "EVALS_UPLOAD is set but HARBOR_API_KEY is not (mint one with: harbor auth login)" +fi + +# Trials are the only way to diagnose a failure: a bare 0.0 cannot distinguish +# a server bug from an agent that misread the prompt. CI sets EVALS_OUT_DIR and +# uploads the tree. +OUT="${EVALS_OUT_DIR:-}" +if [ -n "$OUT" ]; then + mkdir -p "$OUT"; KEEP=1 +else + OUT="$(mktemp -d)"; KEEP=0 +fi +cleanup() { [ "$KEEP" -eq 1 ] || rm -rf "$OUT"; } +trap cleanup EXIT + +# Built from the working tree, so the gate measures the code under review. A +# stale image would pass a PR that breaks the server. +echo "==> Building fred-bench from the working tree" +( cd "$ROOT" && docker build -q -f evals/environment/Dockerfile -t fred-bench . ) + +echo "==> Running $ATTEMPTS attempt(s) per task with claude-code" +# -y auto-confirms harbor's prompts, which would otherwise hang a +# non-interactive runner rather than fail it. +run_args=(--yes --config job.yaml --jobs-dir "$OUT" --job-name "$JOB_NAME" --n-attempts "$ATTEMPTS") +# Off for PR runs: a hub job per push would pile up with nothing to clean them +# up. CI sets it on pushes to main, where keeping the reward/cost/token trend +# for the branch that ships is the point. Uploaded jobs are private by default, +# and the trajectories contain the task instructions and the agent's reasoning. +if [ -n "${EVALS_UPLOAD:-}" ]; then + echo "==> Results will be uploaded to the Harbor hub as $JOB_NAME" + run_args+=(--upload) +fi + +# cd into evals/ because harbor resolves the dataset path relative to the +# working directory. +( cd "$ROOT/evals" && $HARBOR run "${run_args[@]}" ) + +result="$OUT/$JOB_NAME/result.json" +if ! python3 "$ROOT/evals/check_reward.py" "$result" "$JOB_NAME" --min-mean "$MIN_MEAN"; then + # Per trial, named, with the agent's tool calls for anything that lost + # `process`. A bare `cat` of the verifier output prints ten anonymous + # pairs of numbers, which says how many tasks failed and nothing about + # which or why. + echo "--- trials ---" >&2 + python3 "$ROOT/evals/explain_trials.py" "$OUT/$JOB_NAME" >&2 || true + die "the eval gate did not clear $MIN_MEAN" +fi + +echo "==> Eval gate passed." diff --git a/plugins/fred/evals/tasks/find-series-id/environment/Dockerfile b/plugins/fred/evals/tasks/find-series-id/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/find-series-id/instruction.md b/plugins/fred/evals/tasks/find-series-id/instruction.md new file mode 100644 index 0000000..9c87b90 --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/instruction.md @@ -0,0 +1,32 @@ +# Task: Find Series Id + +Find the FRED series ID for the headline US unemployment rate: the monthly, seasonally adjusted one that is the most widely used series of its kind. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"series_id": ""} +``` diff --git a/plugins/fred/evals/tasks/find-series-id/solution/solve.sh b/plugins/fred/evals/tasks/find-series-id/solution/solve.sh new file mode 100755 index 0000000..98a0180 --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/solution/solve.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +cat > "$APP_DIR/answer.json" <<'JSON' +{"series_id": "UNRATE"} +JSON diff --git a/plugins/fred/evals/tasks/find-series-id/task.toml b/plugins/fred/evals/tasks/find-series-id/task.toml new file mode 100644 index 0000000..1cd4cfd --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/find-series-id" +description = "Find the FRED series ID for the headline US unemployment rate: the monthly, seasonally adjusted one that is the most widely used series of its kind." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/find-series-id/tests/outcome/check.py b/plugins/fred/evals/tasks/find-series-id/tests/outcome/check.py new file mode 100644 index 0000000..7521104 --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the text in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. Compared case-insensitively with whitespace +collapsed, since "Billions of Chained 2017 Dollars" and "billions of chained 2017 +dollars" are the same answer and neither is more correct. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "series_id" +EXPECTED = 'UNRATE' + + +def _normal(value: object) -> str: + return " ".join(str(value).split()).strip().lower() + + +@criterion(description="answer.json[series_id] equals UNRATE") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return _normal(data[KEY]) == _normal(EXPECTED) + except Exception: + return False diff --git a/plugins/fred/evals/tasks/find-series-id/tests/process/check.py b/plugins/fred/evals/tasks/find-series-id/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/find-series-id/tests/test.sh b/plugins/fred/evals/tasks/find-series-id/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/find-series-id/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/environment/Dockerfile b/plugins/fred/evals/tasks/gdp-peak-unemployment/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/instruction.md b/plugins/fred/evals/tasks/gdp-peak-unemployment/instruction.md new file mode 100644 index 0000000..80fe74b --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/instruction.md @@ -0,0 +1,32 @@ +# Task: Gdp Peak Unemployment + +Across the observations available for real GDP (GDPC1), find the date on which it was highest, then report the US unemployment rate (UNRATE) for that same date, as a percent. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"unemployment_rate": } +``` diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/solution/solve.sh b/plugins/fred/evals/tasks/gdp-peak-unemployment/solution/solve.sh new file mode 100755 index 0000000..0da0606 --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"unemployment_rate": 4.2}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/task.toml b/plugins/fred/evals/tasks/gdp-peak-unemployment/task.toml new file mode 100644 index 0000000..d882c62 --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/gdp-peak-unemployment" +description = "Across the observations available for real GDP (GDPC1), find the date on which it was highest, then report the US unemployment rate (UNRATE) for that same date, as a percent." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/outcome/check.py b/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/outcome/check.py new file mode 100644 index 0000000..7d16f65 --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "unemployment_rate" +EXPECTED = 4.2 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[unemployment_rate] is within 0.01 of 4.2") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/process/check.py b/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/test.sh b/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/gdp-peak-unemployment/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/inflation-yoy/environment/Dockerfile b/plugins/fred/evals/tasks/inflation-yoy/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/inflation-yoy/instruction.md b/plugins/fred/evals/tasks/inflation-yoy/instruction.md new file mode 100644 index 0000000..452b4f6 --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/instruction.md @@ -0,0 +1,32 @@ +# Task: Inflation Yoy + +For the CPI series CPIAUCSL, find the most recent year-over-year change, as a percent. The server can compute that transformation for you; do not do the arithmetic by hand. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"cpi_yoy_pct": } +``` diff --git a/plugins/fred/evals/tasks/inflation-yoy/solution/solve.sh b/plugins/fred/evals/tasks/inflation-yoy/solution/solve.sh new file mode 100755 index 0000000..1e1e64d --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"cpi_yoy_pct": 2.90557}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/inflation-yoy/task.toml b/plugins/fred/evals/tasks/inflation-yoy/task.toml new file mode 100644 index 0000000..c6bd0ef --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/inflation-yoy" +description = "For the CPI series CPIAUCSL, find the most recent year-over-year change, as a percent. The server can compute that transformation for you; do not do the arithmetic by hand." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/inflation-yoy/tests/outcome/check.py b/plugins/fred/evals/tasks/inflation-yoy/tests/outcome/check.py new file mode 100644 index 0000000..333d173 --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "cpi_yoy_pct" +EXPECTED = 2.90557 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[cpi_yoy_pct] is within 0.01 of 2.90557") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/inflation-yoy/tests/process/check.py b/plugins/fred/evals/tasks/inflation-yoy/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/inflation-yoy/tests/test.sh b/plugins/fred/evals/tasks/inflation-yoy/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/inflation-yoy/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/initial-print/environment/Dockerfile b/plugins/fred/evals/tasks/initial-print/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/initial-print/instruction.md b/plugins/fred/evals/tasks/initial-print/instruction.md new file mode 100644 index 0000000..e85353a --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/instruction.md @@ -0,0 +1,32 @@ +# Task: Initial Print + +Real GDP (GDPC1) for the observation dated 2025-01-01 has since been revised. Find the value as it was FIRST published, not the value it holds today. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"initial_value": } +``` diff --git a/plugins/fred/evals/tasks/initial-print/solution/solve.sh b/plugins/fred/evals/tasks/initial-print/solution/solve.sh new file mode 100755 index 0000000..af497b8 --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"initial_value": 22900.0}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/initial-print/task.toml b/plugins/fred/evals/tasks/initial-print/task.toml new file mode 100644 index 0000000..c4e9659 --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/initial-print" +description = "Real GDP (GDPC1) for the observation dated 2025-01-01 has since been revised. Find the value as it was FIRST published, not the value it holds today." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/initial-print/tests/outcome/check.py b/plugins/fred/evals/tasks/initial-print/tests/outcome/check.py new file mode 100644 index 0000000..1ee6c9a --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "initial_value" +EXPECTED = 22900.0 +TOLERANCE = 0.5 + + +@criterion(description="answer.json[initial_value] is within 0.5 of 22900.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/initial-print/tests/process/check.py b/plugins/fred/evals/tasks/initial-print/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/initial-print/tests/test.sh b/plugins/fred/evals/tasks/initial-print/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/initial-print/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/next-release/environment/Dockerfile b/plugins/fred/evals/tasks/next-release/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/next-release/instruction.md b/plugins/fred/evals/tasks/next-release/instruction.md new file mode 100644 index 0000000..32f4f9c --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/instruction.md @@ -0,0 +1,32 @@ +# Task: Next Release + +The Employment Situation release has FRED release id 50. Find the date of its NEXT scheduled release, that is, the first one still in the future. Report it as YYYY-MM-DD. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"next_release_date": ""} +``` diff --git a/plugins/fred/evals/tasks/next-release/solution/solve.sh b/plugins/fred/evals/tasks/next-release/solution/solve.sh new file mode 100755 index 0000000..9c3a283 --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/solution/solve.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Oracle for the calendar task. The expected date moves with the clock, so the oracle +# computes it the same way the verifier does rather than echoing a literal. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +python3 - "$APP_DIR/answer.json" <<'PY' +import json, sys +from datetime import date, timedelta +answer = (date.today() + timedelta(days=9)).isoformat() +with open(sys.argv[1], "w") as fh: + json.dump({"next_release_date": answer}, fh) +PY diff --git a/plugins/fred/evals/tasks/next-release/task.toml b/plugins/fred/evals/tasks/next-release/task.toml new file mode 100644 index 0000000..694482e --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/next-release" +description = "The Employment Situation release has FRED release id 50. Find the date of its NEXT scheduled release, that is, the first one still in the future. Report it as YYYY-MM-DD." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/next-release/tests/outcome/check.py b/plugins/fred/evals/tasks/next-release/tests/outcome/check.py new file mode 100644 index 0000000..34aff66 --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/tests/outcome/check.py @@ -0,0 +1,36 @@ +"""`outcome` reward: the next release date for release 50. + +Generated by evals/generate_tasks.py. The fixture places release dates relative to the +current day, so the expected answer is computed here rather than baked in: a literal +date would stop being the right answer tomorrow. +""" + +import json +from datetime import date, timedelta +from pathlib import Path + +from rewardkit import criterion + +KEY = "next_release_date" +OFFSET_DAYS = 9 + + +def _accepted() -> set: + """The expected date, plus the one for yesterday. + + The agent and the verifier are two processes, and a trial that starts just before + midnight is graded on the next day against a fixture that has moved with it. + Accepting both is a tolerance on the clock, the same as the numeric tasks carry a + tolerance on the value. + """ + today = date.today() + return {(day + timedelta(days=OFFSET_DAYS)).isoformat() for day in (today, today - timedelta(days=1))} + + +@criterion(description="answer.json[next_release_date] is the next scheduled release date") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return str(data[KEY]).strip() in _accepted() + except Exception: + return False diff --git a/plugins/fred/evals/tasks/next-release/tests/process/check.py b/plugins/fred/evals/tasks/next-release/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/next-release/tests/test.sh b/plugins/fred/evals/tasks/next-release/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/next-release/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/rate-history-max/environment/Dockerfile b/plugins/fred/evals/tasks/rate-history-max/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/rate-history-max/instruction.md b/plugins/fred/evals/tasks/rate-history-max/instruction.md new file mode 100644 index 0000000..a6a6fe4 --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/instruction.md @@ -0,0 +1,32 @@ +# Task: Rate History Max + +For the 10-year Treasury series DGS10, find the highest value it has ever reached over its entire history in this dataset. Be careful: a long daily series is returned as a sample of its points, and the highest point is not necessarily among the ones you get back. The tool reports the true figure alongside the points. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"max_yield": } +``` diff --git a/plugins/fred/evals/tasks/rate-history-max/solution/solve.sh b/plugins/fred/evals/tasks/rate-history-max/solution/solve.sh new file mode 100755 index 0000000..d46b0f3 --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"max_yield": 300.0}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/rate-history-max/task.toml b/plugins/fred/evals/tasks/rate-history-max/task.toml new file mode 100644 index 0000000..0513879 --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/rate-history-max" +description = "For the 10-year Treasury series DGS10, find the highest value it has ever reached over its entire history in this dataset. Be careful: a long daily series is returned as a sample of its points, and the highest point is not necessarily among the ones you get back. The tool reports the true figure alongside the points." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/rate-history-max/tests/outcome/check.py b/plugins/fred/evals/tasks/rate-history-max/tests/outcome/check.py new file mode 100644 index 0000000..ba34f68 --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "max_yield" +EXPECTED = 300.0 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[max_yield] is within 0.01 of 300.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/rate-history-max/tests/process/check.py b/plugins/fred/evals/tasks/rate-history-max/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/rate-history-max/tests/test.sh b/plugins/fred/evals/tasks/rate-history-max/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/rate-history-max/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/release-series-count/environment/Dockerfile b/plugins/fred/evals/tasks/release-series-count/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/release-series-count/instruction.md b/plugins/fred/evals/tasks/release-series-count/instruction.md new file mode 100644 index 0000000..afadfb5 --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/instruction.md @@ -0,0 +1,32 @@ +# Task: Release Series Count + +The Employment Situation release has FRED release id 50. Count how many of the series it publishes are monthly. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"series_count": } +``` diff --git a/plugins/fred/evals/tasks/release-series-count/solution/solve.sh b/plugins/fred/evals/tasks/release-series-count/solution/solve.sh new file mode 100755 index 0000000..31a619f --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"series_count": 3.0}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/release-series-count/task.toml b/plugins/fred/evals/tasks/release-series-count/task.toml new file mode 100644 index 0000000..13622f7 --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/release-series-count" +description = "The Employment Situation release has FRED release id 50. Count how many of the series it publishes are monthly." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/release-series-count/tests/outcome/check.py b/plugins/fred/evals/tasks/release-series-count/tests/outcome/check.py new file mode 100644 index 0000000..0a3f493 --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "series_count" +EXPECTED = 3.0 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[series_count] is within 0.01 of 3.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/release-series-count/tests/process/check.py b/plugins/fred/evals/tasks/release-series-count/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/release-series-count/tests/test.sh b/plugins/fred/evals/tasks/release-series-count/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/release-series-count/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/revision-count/environment/Dockerfile b/plugins/fred/evals/tasks/revision-count/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/revision-count/instruction.md b/plugins/fred/evals/tasks/revision-count/instruction.md new file mode 100644 index 0000000..ed95b4f --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/instruction.md @@ -0,0 +1,32 @@ +# Task: Revision Count + +Across the observations available for real GDP (GDPC1), count how many have been revised since they were first published, that is, how many now hold a different value than the one first reported. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"revised_count": } +``` diff --git a/plugins/fred/evals/tasks/revision-count/solution/solve.sh b/plugins/fred/evals/tasks/revision-count/solution/solve.sh new file mode 100755 index 0000000..04ecd47 --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"revised_count": 1.0}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/revision-count/task.toml b/plugins/fred/evals/tasks/revision-count/task.toml new file mode 100644 index 0000000..4462b69 --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/revision-count" +description = "Across the observations available for real GDP (GDPC1), count how many have been revised since they were first published, that is, how many now hold a different value than the one first reported." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/revision-count/tests/outcome/check.py b/plugins/fred/evals/tasks/revision-count/tests/outcome/check.py new file mode 100644 index 0000000..de1d08f --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "revised_count" +EXPECTED = 1.0 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[revised_count] is within 0.01 of 1.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/revision-count/tests/process/check.py b/plugins/fred/evals/tasks/revision-count/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/revision-count/tests/test.sh b/plugins/fred/evals/tasks/revision-count/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/revision-count/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/series-units/environment/Dockerfile b/plugins/fred/evals/tasks/series-units/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/series-units/instruction.md b/plugins/fred/evals/tasks/series-units/instruction.md new file mode 100644 index 0000000..5d9897c --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/instruction.md @@ -0,0 +1,32 @@ +# Task: Series Units + +What are the units of the real GDP series GDPC1? Report the full units description exactly as FRED gives it. + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"units": ""} +``` diff --git a/plugins/fred/evals/tasks/series-units/solution/solve.sh b/plugins/fred/evals/tasks/series-units/solution/solve.sh new file mode 100755 index 0000000..60a5625 --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/solution/solve.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +cat > "$APP_DIR/answer.json" <<'JSON' +{"units": "Billions of Chained 2017 Dollars"} +JSON diff --git a/plugins/fred/evals/tasks/series-units/task.toml b/plugins/fred/evals/tasks/series-units/task.toml new file mode 100644 index 0000000..3d99531 --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/series-units" +description = "What are the units of the real GDP series GDPC1? Report the full units description exactly as FRED gives it." + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/series-units/tests/outcome/check.py b/plugins/fred/evals/tasks/series-units/tests/outcome/check.py new file mode 100644 index 0000000..730d80c --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the text in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. Compared case-insensitively with whitespace +collapsed, since "Billions of Chained 2017 Dollars" and "billions of chained 2017 +dollars" are the same answer and neither is more correct. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "units" +EXPECTED = 'Billions of Chained 2017 Dollars' + + +def _normal(value: object) -> str: + return " ".join(str(value).split()).strip().lower() + + +@criterion(description="answer.json[units] equals Billions of Chained 2017 Dollars") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return _normal(data[KEY]) == _normal(EXPECTED) + except Exception: + return False diff --git a/plugins/fred/evals/tasks/series-units/tests/process/check.py b/plugins/fred/evals/tasks/series-units/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/series-units/tests/test.sh b/plugins/fred/evals/tasks/series-units/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/series-units/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/tasks/unemployment-latest/environment/Dockerfile b/plugins/fred/evals/tasks/unemployment-latest/environment/Dockerfile new file mode 100644 index 0000000..fd556cf --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/environment/Dockerfile @@ -0,0 +1 @@ +FROM fred-bench diff --git a/plugins/fred/evals/tasks/unemployment-latest/instruction.md b/plugins/fred/evals/tasks/unemployment-latest/instruction.md new file mode 100644 index 0000000..3a3abad --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/instruction.md @@ -0,0 +1,32 @@ +# Task: Unemployment Latest + +What is the most recent US unemployment rate (series UNRATE), as a percent? + +Use the FRED MCP tools. They are named `mcp__fred__*`, and the server behind them is +already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__fred__get_observations`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. + +The work has to go through the tools. Do not call the data provider's HTTP API directly, +do not read or edit the server's source or its test fixtures, and do not import its Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another FRED tool. Do +not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: + +```json +{"unemployment_rate": } +``` diff --git a/plugins/fred/evals/tasks/unemployment-latest/solution/solve.sh b/plugins/fred/evals/tasks/unemployment-latest/solution/solve.sh new file mode 100755 index 0000000..794fd85 --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"unemployment_rate": 4.3}' > "$APP_DIR/answer.json" diff --git a/plugins/fred/evals/tasks/unemployment-latest/task.toml b/plugins/fred/evals/tasks/unemployment-latest/task.toml new file mode 100644 index 0000000..806ee1a --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/task.toml @@ -0,0 +1,16 @@ +[task] +name = "fred-mcp/unemployment-latest" +description = "What is the most recent US unemployment rate (series UNRATE), as a percent?" + +[metadata] +suite = "fred-mcp" + +[environment] +docker_image = "fred-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/fred/evals/tasks/unemployment-latest/tests/outcome/check.py b/plugins/fred/evals/tasks/unemployment-latest/tests/outcome/check.py new file mode 100644 index 0000000..deb396e --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the mock +fixtures by the same shaping code the server uses, so it cannot drift from what the +agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "unemployment_rate" +EXPECTED = 4.3 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[unemployment_rate] is within 0.01 of 4.3") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than erroring the + # trial: an agent that writes nothing has failed the task, which is a verdict, + # not a harness fault. + return False diff --git a/plugins/fred/evals/tasks/unemployment-latest/tests/process/check.py b/plugins/fred/evals/tasks/unemployment-latest/tests/process/check.py new file mode 100644 index 0000000..5f061b4 --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/tests/process/check.py @@ -0,0 +1,128 @@ +"""`process` reward: the answer came through the fred MCP server. + +There are three ways round this server, and `outcome` cannot see any of them: + + 1. the mock FRED API, reachable over plain HTTP inside the container + 2. the fixtures on disk, which hold every expected answer in plain Python + 3. the real FRED API, reachable because the agent needs the network for its own model + +A tastytrade gate run took route 1 and scored a clean 1.0 on outcome alone. This reward +is what catches all three. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") + + +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. + + `path` matters: rewardkit's own trajectory helpers default to /logs/trajectory.json + while Harbor agents write /logs/agent/trajectory.json, and a missing file scores 0 + silently rather than erroring. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir scan drops + any jsonl whose path contains a `subagents/` component, and modern Claude Code writes + each subagent's transcript there. A call the agent delegated therefore leaves an + `Agent` entry in the trajectory and no tool. + + Reading the raw transcripts makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also seeing a + delegated `curl` would turn "ask a subagent" into an invisible bypass, which is the + hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + A list rather than a generator so every criterion can fail closed on an empty + trajectory. A "did not bypass" check is vacuously true when there are no calls at + all, which would hand a no-op run half of `process`; no record means no evidence the + intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__fred__" +# The three ways round the server. Matched against tool arguments, so it catches Bash, +# Read, and Edit alike without enumerating tool names. +# +# The port alone, not host:port. The mock binds every interface, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; enumerating two +# spellings lets the other three through, and a bypass that scores as good behaviour is +# worse than no check. Nothing else in the image listens on that port. +# +# The hostname of the real API is here because this benchmark runs with the network up +# (the agent needs it to reach its own model), which tastytrade's mock-only setup did not +# have to consider. +BYPASS = re.compile(r":8080\b|\bfred_api\b|stlouisfed", re.IGNORECASE) + + +@criterion(description="Agent called a fred MCP tool") +def used_mcp_server(workspace: Path) -> bool: + """Anywhere in the run, subagents included -- see `_session_calls`. + + Whether the top-level agent placed the call or handed it to a delegate is the + harness's routing decision, not a fact about this plugin. The question the gate asks + is whether a real agent can drive the server to the answer, and a delegated call is + that. + """ + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the data directly") +def no_direct_data_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/fred/evals/tasks/unemployment-latest/tests/test.sh b/plugins/fred/evals/tasks/unemployment-latest/tests/test.sh new file mode 100755 index 0000000..a833d5e --- /dev/null +++ b/plugins/fred/evals/tasks/unemployment-latest/tests/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server +# +# `outcome` alone cannot gate this plugin. The mock FRED API is reachable over +# plain HTTP from inside the container, its fixtures sit on disk in plain Python, +# and the real FRED API is reachable over the network. An agent that ignores the +# MCP entirely can still produce the right answer. `process` is what makes these +# MCP evals rather than answer-matching. +rewardkit /tests diff --git a/plugins/fred/evals/validate_in_container.sh b/plugins/fred/evals/validate_in_container.sh new file mode 100755 index 0000000..8a6cc8e --- /dev/null +++ b/plugins/fred/evals/validate_in_container.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Runs inside fred-bench, invoked by validate_local.sh. Scores every task's real +# rewardkit verifier against synthetic trajectories and asserts the reward matrix. +# +# case answer trajectory outcome process +# ------------------ ------- ---------------------------------- ------- ------- +# solved oracle called an MCP tool 1 1 +# empty none none 0 0 +# bypassed-port oracle curled the local mock 1 0 +# bypassed-fixture oracle read the fixtures off disk 1 0 +# bypassed-real oracle curled the real FRED API 1 0 +# delegated oracle subagent called an MCP tool 1 1 +# delegated-bypass oracle subagent curled the local mock 1 0 +# +# The bypass rows are the point, and they are what `harbor run -a oracle` cannot tell +# you. This plugin has three ways round the server rather than tastytrade's two, so +# each gets its own row: +# +# port the mock binds every interface, so it answers on localhost, 127.0.0.1, +# 0.0.0.0, [::1] and the container's hostname. This row deliberately uses +# 0.0.0.0, the spelling a hostname list would miss. +# fixture every expected answer sits on disk in plain Python. Reading it needs no +# network at all. +# real the benchmark runs with the network up, because the agent needs it to +# reach its own model, so api.stlouisfed.org is reachable too. tastytrade's +# mock-only setup never had to consider this one. +# +# The delegated pair covers the blind spot behind all three: harbor's trajectory holds +# the main session only, so a call handed to a subagent shows up as an `Agent` entry +# and no tool. Both halves are asserted together, because crediting a delegated MCP +# call while missing a delegated curl would make "ask a subagent" an invisible bypass. +# +# Expects the repo at /work. Nothing here calls a model or the network. +set -uo pipefail + +TASKS=/work/evals/tasks +pass=0 +fail=0 + +mcp_good='{"steps":[{"tool_calls":[{"function_name":"mcp__fred__get_observations","arguments":{"series_ids":"UNRATE"}}]}]}' +bypass_port='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"curl -s http://0.0.0.0:8080/fred/series?series_id=UNRATE"}}]}]}' +bypass_fixture='{"steps":[{"tool_calls":[{"function_name":"Read","arguments":{"file_path":"/opt/fred/tests/fixtures/fred_api.py"}}]}]}' +bypass_real='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"curl -s https://api.stlouisfed.org/fred/series/observations?series_id=UNRATE"}}]}]}' + +# What the top-level agent's trajectory looks like when it delegates: an Agent call and +# nothing else. Paired with a subagent transcript below. +delegating='{"steps":[{"tool_calls":[{"function_name":"Agent","arguments":{"description":"Look up the answer"}}]}]}' +# Subagent transcripts are Claude Code session lines, not harbor trajectories: +# type/message.content[] with tool_use blocks carrying `name` and `input`. +sub_mcp='{"type":"assistant","isSidechain":true,"message":{"content":[{"type":"tool_use","id":"t1","name":"mcp__fred__get_observations","input":{"series_ids":"UNRATE"}}]}}' +sub_bypass='{"type":"assistant","isSidechain":true,"message":{"content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"curl -s http://localhost:8080/fred/series"}}]}}' + +# Score one task against one trajectory, optionally with a subagent transcript. +# Echoes " ". +# +# $4 is a Claude Code session line placed where a real subagent's would land, at +# sessions/projects///subagents/. That path is the whole point: harbor's +# session scan skips anything under `subagents/`, so a call written there is absent from +# trajectory.json by construction, exactly as it is in a real delegated run. +score() { + local task=$1 trajectory=$2 solved=$3 subagent=${4:-} + local work + work="$(mktemp -d)" + mkdir -p "$work/app" "$work/logs/agent" "$work/logs/verifier" + + if [ "$solved" = "yes" ]; then + APP_DIR="$work/app" bash "$TASKS/$task/solution/solve.sh" > /dev/null 2>&1 + fi + if [ -n "$trajectory" ]; then + printf '%s' "$trajectory" > "$work/logs/agent/trajectory.json" + fi + if [ -n "$subagent" ]; then + local subdir="$work/logs/agent/sessions/projects/app/sess/subagents" + mkdir -p "$subdir" + printf '%s\n' "$subagent" > "$subdir/sub.jsonl" + fi + + # rewardkit reads the trajectory from an absolute path baked into the check, so + # /logs has to be the real one rather than a flag. + rm -rf /logs && ln -s "$work/logs" /logs + rewardkit "$TASKS/$task/tests" --workspace "$work/app" \ + --output "$work/logs/verifier/reward.json" > /dev/null 2>&1 + + python3 - "$work/logs/verifier/reward.json" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception: + print("err err"); raise SystemExit +print(f"{d.get('outcome', 'missing')} {d.get('process', 'missing')}") +PY + rm -rf "$work" +} + +expect() { + local task=$1 case_name=$2 got=$3 want=$4 + if [ "$got" = "$want" ]; then + pass=$((pass + 1)) + else + fail=$((fail + 1)) + echo "FAIL $task [$case_name]: expected (outcome process) = ($want), got ($got)" + fi +} + +for dir in "$TASKS"/*/; do + task="$(basename "$dir")" + [ -f "$dir/tests/test.sh" ] || continue + + expect "$task" solved "$(score "$task" "$mcp_good" yes)" "1.0 1.0" + expect "$task" empty "$(score "$task" '' no)" "0.0 0.0" + expect "$task" bypassed-port "$(score "$task" "$bypass_port" yes)" "1.0 0.0" + expect "$task" bypassed-fixture "$(score "$task" "$bypass_fixture" yes)" "1.0 0.0" + expect "$task" bypassed-real "$(score "$task" "$bypass_real" yes)" "1.0 0.0" + expect "$task" delegated "$(score "$task" "$delegating" yes "$sub_mcp")" "1.0 1.0" + expect "$task" delegated-bypass "$(score "$task" "$delegating" yes "$sub_bypass")" "1.0 0.0" +done + +echo +echo "$pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/plugins/fred/evals/validate_local.sh b/plugins/fred/evals/validate_local.sh new file mode 100755 index 0000000..ea60200 --- /dev/null +++ b/plugins/fred/evals/validate_local.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Validate every task's verifier WITHOUT Harbor and without a model: score the real +# rewardkit checks against synthetic trajectories and assert the reward matrix (see +# validate_in_container.sh for the table). This is the local stand-in for +# `harbor run -a oracle`, and it catches a verifier that accepts a wrong answer or one +# that cannot tell the intended route from a bypass. +# +# Runs in the bench image rather than on the host: rewardkit scores these checks and +# does not build on macOS, where its litellm dependency wants a newer rustc than ships +# there. Using the same image CI uses also means the verifier under test is the one that +# will really grade a gate run. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +die() { echo "error: $1" >&2; exit 1; } + +docker info > /dev/null 2>&1 \ + || die "docker is not running (the verifiers need rewardkit, which lives in the bench image)" + +# Rebuilt every time: the checks under test are generated, so scoring a stale image +# would report on the previous generation of tasks. +echo "==> Building fred-bench" +docker build -q -f "$ROOT/evals/environment/Dockerfile" -t fred-bench "$ROOT" > /dev/null + +echo "==> Scoring every verifier: solved / empty / three bypasses / delegated" +docker run --rm -v "$ROOT:/work:ro" fred-bench bash /work/evals/validate_in_container.sh diff --git a/plugins/fred/src/client.py b/plugins/fred/src/client.py index b49e0e8..4ac1369 100644 --- a/plugins/fred/src/client.py +++ b/plugins/fred/src/client.py @@ -108,7 +108,9 @@ def __init__( # in the tool layer) never raises; a missing key should surface as a guided # error from the tool that needed it, not as a server that will not start. self._api_key = api_key - self.base_url = (base_url or DEFAULT_BASE_URL).rstrip("/") + # FRED_BASE_URL is what points the server at a local mock, which is how the + # eval benchmark runs without a key and without touching the real API. + self.base_url = (base_url or os.environ.get("FRED_BASE_URL") or DEFAULT_BASE_URL).strip().rstrip("/") self._transport = transport self._http: httpx.AsyncClient | None = None diff --git a/plugins/fred/tests/fixtures/fred_api.py b/plugins/fred/tests/fixtures/fred_api.py index d56118c..3e2ad43 100644 --- a/plugins/fred/tests/fixtures/fred_api.py +++ b/plugins/fred/tests/fixtures/fred_api.py @@ -1,19 +1,31 @@ -"""A mock FRED API, as an httpx transport. +"""A mock FRED API. + +One routing function, three callers: the unit tests drive it through an httpx +transport, the integration tests drive the real MCP tools through the same transport, +and the eval benchmark serves it over HTTP inside the container. Everything scores +against identical fixture behaviour, so a benchmark answer can never disagree with a +test answer. Responses are trimmed captures from the real API, so the shapes the shaping layer is -asserted against are FRED's rather than ones invented to match the code. The whole -thing is a routing function because that is all the integration tests need: no server, -no port, no ASGI app. +asserted against are FRED's rather than ones invented to match the code. + +Serve it (this is what the benchmark container runs): -Every handler records the query it was called with, which is how the tests check that -the tools send the parameters they claim to (the real-time window on a vintage request, -the filter_variable pairing, the popularity ordering). + python -m tests.fixtures.fred_api 8080 """ -from datetime import date +import json +import math +from datetime import date, timedelta import httpx +# The clock the tests pin. Release dates are generated relative to a clock rather than +# written down, because the calendar's whole job is "what came out and what is next": +# fixed dates stop straddling today the moment the fixture ages, and the benchmark +# runs on whatever day CI happens to run. +FIXED_TODAY = date(2026, 8, 5) + UNRATE = { "id": "UNRATE", "realtime_start": "2026-08-05", @@ -71,7 +83,21 @@ "popularity": 90, } -SERIES = {s["id"]: s for s in (UNRATE, CPIAUCSL, UNRATENSA, GDPC1)} +DGS10 = { + **UNRATE, + "id": "DGS10", + "title": "Market Yield on U.S. Treasury Securities at 10-Year Constant Maturity", + "frequency": "Daily", + "frequency_short": "D", + "units": "Percent", + "popularity": 88, +} + +SERIES = {s["id"]: s for s in (UNRATE, CPIAUCSL, UNRATENSA, GDPC1, DGS10)} + +# Every series in the Employment Situation release (id 50). Three monthly, one +# quarterly, so a frequency filter over a release has something to actually filter. +RELEASE_50_SERIES = [UNRATE, CPIAUCSL, UNRATENSA, GDPC1] # Monthly, with a "." in the middle: FRED's missing-value marker, not a null. UNRATE_OBS = [ @@ -90,38 +116,58 @@ ("2025-04-01", "23150.5"), ] -# A long daily series for the downsampling tests. The knots put the peak and the trough -# in the *interior*, away from the first and last points that downsampling always keeps. -# That is the whole point: a summary computed after thinning would report the extremes -# of the sample, and with extremes at the endpoints the test could not tell the -# difference. -_DAILY_KNOTS = [(0, 200.0), (80, 300.0), (200, 50.0), (365, 180.0)] + +def _monthly(start: date, count: int, first: float, step: float) -> list[tuple[str, str]]: + rows = [] + for i in range(count): + month = start.month - 1 + i + stamp = date(start.year + month // 12, month % 12 + 1, 1) + rows.append((stamp.isoformat(), f"{first + step * i:.4f}")) + return rows + + +# 26 months, which is what makes a year-over-year transform expressible at all: pc1 +# needs an observation twelve months back, and a six-point series has none. +CPI_OBS = _monthly(date(2024, 1, 1), 26, 100.0, 0.25) + + +# A long daily series, five years of it, for the downsampling tests and for the +# `rate-history-max` eval task. +# +# The extremes are single-day spikes at indices that downsampling does not sample. That +# is deliberate and it is the whole point: the summary is computed over every +# observation and only the point list is thinned, so the true extremes have to be +# reachable from the summary and *unreachable* from the points. An earlier version used +# a broad triangular peak, and the sampling grid happened to land exactly on it, so a +# lazy agent reading the returned points got the right answer and the task proved +# nothing. `test_the_extremes_are_not_in_the_returned_points` now fails if that +# regresses. +_DAILY_DAYS = 1827 +_SPIKE_INDEX, _TROUGH_INDEX = 900, 1200 +DAILY_MIN, DAILY_MAX = 20.0, 300.0 def _daily_series() -> list[tuple[str, str]]: - values: list[float] = [] - for (i0, v0), (i1, v1) in zip(_DAILY_KNOTS, _DAILY_KNOTS[1:]): - values.extend(v0 + (v1 - v0) * (i - i0) / (i1 - i0) for i in range(i0, i1)) - values.append(_DAILY_KNOTS[-1][1]) + # A slow wave for the body of the series, well inside the extremes. + values = [130 + 70 * math.sin(2 * math.pi * i / 900) for i in range(_DAILY_DAYS)] + values[_SPIKE_INDEX] = DAILY_MAX + values[_TROUGH_INDEX] = DAILY_MIN first = date(2020, 1, 1).toordinal() return [(date.fromordinal(first + i).isoformat(), f"{v:.4f}") for i, v in enumerate(values)] DAILY_OBS = _daily_series() -DAILY_MIN, DAILY_MAX = 50.0, 300.0 -OBSERVATIONS = {"UNRATE": UNRATE_OBS, "GDPC1": GDPC1_OBS, "CPIAUCSL": UNRATE_OBS, "DGS10": DAILY_OBS} +OBSERVATIONS = {"UNRATE": UNRATE_OBS, "GDPC1": GDPC1_OBS, "CPIAUCSL": CPI_OBS, "DGS10": DAILY_OBS} -# What FRED returns for output_type=4: the value as first published, with the realtime -# window showing when that print was current. GDPC1 gets revised, UNRATE does not. +# What FRED returns for output_type=4: the value as first published. GDPC1's first +# quarter was revised; its second stands as published, and UNRATE is never revised. INITIAL_OBS = { - # 2025-01-01 was revised (22900 -> 23000); 2025-04-01 stands as first published. - # One of each, so "revised" counting is tested against a mix rather than a - # uniformly-revised series where an off-by-one would pass. "GDPC1": [("2025-01-01", "22900.0"), ("2025-04-01", "23150.5")], "UNRATE": UNRATE_OBS, - "CPIAUCSL": UNRATE_OBS, + "CPIAUCSL": CPI_OBS, + "DGS10": DAILY_OBS, } # One column per vintage, the output_type=2 shape. Six vintages, one real revision: @@ -138,20 +184,233 @@ def _daily_series() -> list[tuple[str, str]]: VINTAGE_DATES = ["2025-09-25", "2025-08-28", "2025-07-30", "2025-06-25", "2025-05-28", "2025-04-30"] -# Dates straddle TODAY so the released/upcoming split has something on both sides. -TODAY = "2026-08-05" -RELEASE_DATES = [ - {"release_id": 50, "release_name": "Employment Situation", "date": "2026-08-01"}, - {"release_id": 10, "release_name": "Consumer Price Index", "date": "2026-08-05"}, - {"release_id": 50, "release_name": "Employment Situation", "date": "2026-08-07"}, - {"release_id": 10, "release_name": "Consumer Price Index", "date": "2026-08-12"}, -] +RELEASES = { + 50: {"id": 50, "name": "Employment Situation", "link": "http://www.bls.gov/ces/", "press_release": True}, + 10: {"id": 10, "name": "Consumer Price Index", "link": "http://www.bls.gov/cpi/", "press_release": True}, +} + +# Offsets in days from "today". Chosen so the calendar's default window (7 days back, +# 14 forward) holds three released dates including one dated exactly today, and two +# upcoming ones, with dates outside the window on both sides to prove it filters. +RELEASE_OFFSETS = {50: (-33, -5, 9, 37), 10: (-12, -2, 0, 12, 40)} +# The offset `next-release` expects for release 50: the first one still ahead. +NEXT_RELEASE_50_OFFSET = 9 + + +def release_dates(today: date) -> list[dict]: + """Every release date, ascending, relative to the given day.""" + rows = [ + { + "release_id": release_id, + "release_name": RELEASES[release_id]["name"], + "date": (today + timedelta(days=offset)).isoformat(), + } + for release_id, offsets in RELEASE_OFFSETS.items() + for offset in offsets + ] + return sorted(rows, key=lambda r: (r["date"], r["release_id"])) + + +# --- units transforms ---------------------------------------------------------------- +# +# The mock applies these for real. Echoing the requested units while returning raw +# levels would make `units="yoy"` indistinguishable from not passing units at all, +# which is exactly the correction the eval task exists to measure. + + +def _a_year_before(stamp: str) -> str: + day = date.fromisoformat(stamp) + try: + return day.replace(year=day.year - 1).isoformat() + except ValueError: # 29 February + return day.replace(year=day.year - 1, day=28).isoformat() + + +def transform(rows: list[tuple[str, str]], units: str) -> list[tuple[str, str]]: + """Apply a FRED units code. Values that cannot be computed become ".", as FRED does.""" + if units in ("", "lin"): + return rows + + values = {stamp: (None if raw == "." else float(raw)) for stamp, raw in rows} + order = [stamp for stamp, _ in rows] + previous = {stamp: order[i - 1] if i else None for i, stamp in enumerate(order)} + + out: list[tuple[str, str]] = [] + for stamp, _ in rows: + current = values.get(stamp) + base_key = _a_year_before(stamp) if units in ("pc1", "ch1") else previous[stamp] + base = values.get(base_key) if base_key else None + + if current is None or base is None: + out.append((stamp, ".")) + continue + if units in ("chg", "ch1"): + out.append((stamp, f"{current - base:.5f}")) + elif units in ("pch", "pc1"): + out.append((stamp, "." if base == 0 else f"{(current - base) / base * 100:.5f}")) + else: # anything else is returned as published rather than silently faked + out.append((stamp, f"{current:.5f}")) + return out + + +# --- routing ------------------------------------------------------------------------- + + +def _error(status: int, message: str) -> tuple[int, dict]: + return status, {"error_code": status, "error_message": message} + + +def _tags_for(series: dict) -> set[str]: + """The freq and seas tags FRED would carry for a series, derived from its fields.""" + return {series["frequency"].lower(), "nsa" if series["seasonal_adjustment_short"] == "NSA" else "sa"} + + +def _limited(rows: list, params: dict[str, str]) -> list: + if params.get("sort_order") == "desc": + rows = sorted(rows, reverse=True) + return rows[: int(params.get("limit", 100000))] + + +def route(path: str, params: dict[str, str], today: date | None = None) -> tuple[int, dict]: + """Answer one FRED request. Pure: no I/O, no globals, no clock of its own.""" + now = today or date.today() + path = path.rstrip("/") + + # Exact, not endswith: /release/series and /category/series also end in "/series". + if path.endswith("/fred/series"): + return _one_series(params.get("series_id", "")) + if path.endswith("/series/observations"): + return _observations(params, now) + if path.endswith("/series/vintagedates"): + return 200, {"count": len(VINTAGE_DATES), "vintage_dates": VINTAGE_DATES[: int(params.get("limit", 100))]} + if path.endswith("/series/search"): + return _series_list(params, [UNRATE, UNRATENSA, CPIAUCSL, GDPC1, DGS10]) + if path.endswith("/release/series"): + pool = RELEASE_50_SERIES if params.get("release_id") == "50" else [] + return _series_list(params, pool) + if path.endswith("/category/series"): + return _series_list(params, [UNRATE, CPIAUCSL]) + if path.endswith("/series/release"): + return 200, {"releases": [RELEASES[50]]} + if path.endswith("/series/categories"): + return 200, { + "categories": [ + {"id": 32447, "name": "Unemployment Rate", "parent_id": 12, "notes": "Unemployed over labor force."} + ] + } + if path.endswith("/series/tags"): + return 200, { + "count": 2, + "tags": [ + {"name": "headline figure", "group_id": "gen", "notes": "", "popularity": 51}, + {"name": "monthly", "group_id": "freq", "notes": "", "popularity": 93}, + ], + } + if path.endswith("/releases/dates"): + return _release_dates(params, release_dates(now), now) + if path.endswith("/release/dates"): + # FRED omits release_name here, unlike /releases/dates. + rows = [ + {"release_id": r["release_id"], "date": r["date"]} + for r in release_dates(now) + if str(r["release_id"]) == params.get("release_id") + ] + return _release_dates(params, rows, now) + if path.endswith("/fred/release"): + release = RELEASES.get(int(params.get("release_id", 0) or 0)) + if release is None: + return _error(400, "Bad Request. The release does not exist.") + return 200, {"releases": [release]} + return _error(404, f"Not Found. No handler for {path}.") + + +def _one_series(series_id: str) -> tuple[int, dict]: + if series_id not in SERIES: + return _error(400, "Bad Request. The series does not exist.") + return 200, {"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "seriess": [SERIES[series_id]]} + + +def _observations(params: dict[str, str], now: date) -> tuple[int, dict]: + series_id = params.get("series_id", "") + if series_id not in OBSERVATIONS: + return _error(400, "Bad Request. The series does not exist.") + + output_type = params.get("output_type", "1") + if output_type in ("2", "4"): + # FRED's own behaviour, and the reason get_revisions exists: without a + # real-time window spanning the record, a vintage request fails. + if params.get("realtime_start") != "1776-07-04": + return _error( + 400, + "Bad Request. No vintage dates exist for the specified real-time period: " + f"{now.isoformat()} to {now.isoformat()}.", + ) + if output_type == "2": + row = VINTAGE_ROW if params.get("observation_start") == VINTAGE_ROW["date"] else None + return 200, {"observations": [row] if row else []} + return _payload(_limited(INITIAL_OBS.get(series_id, []), params), params) + + rows = transform(OBSERVATIONS[series_id], params.get("units", "lin")) + start, end = params.get("observation_start"), params.get("observation_end") + if start: + rows = [r for r in rows if r[0] >= start] + if end: + rows = [r for r in rows if r[0] <= end] + return _payload(_limited(rows, params), params) + + +def _payload(rows: list[tuple[str, str]], params: dict[str, str]) -> tuple[int, dict]: + # Real-time fields carry the same value on every row, which is the redundancy the + # columnar shaping exists to remove; the fixture reproduces it faithfully. + stamp = "2026-08-05" + return 200, { + "realtime_start": stamp, + "realtime_end": stamp, + "units": params.get("units", "lin"), + "count": len(rows), + "observations": [{"realtime_start": stamp, "realtime_end": stamp, "date": d, "value": v} for d, v in rows], + } + + +def _series_list(params: dict[str, str], pool: list[dict]) -> tuple[int, dict]: + rows = list(pool) + for tag in filter(None, params.get("tag_names", "").split(";")): + rows = [r for r in rows if tag in _tags_for(r)] + + order_by = params.get("order_by", "series_id") + reverse = params.get("sort_order", "asc") == "desc" + if order_by in {"popularity", "group_popularity"}: + rows.sort(key=lambda r: r.get(order_by, 0), reverse=reverse) + else: + rows.sort(key=lambda r: str(r.get(order_by, r["id"])), reverse=reverse) + + total = len(rows) + limit = int(params.get("limit", 1000)) + return 200, {"count": total, "offset": 0, "limit": limit, "seriess": rows[:limit]} + + +def _release_dates(params: dict[str, str], pool: list[dict], now: date) -> tuple[int, dict]: + rows = list(pool) + start, end = params.get("realtime_start"), params.get("realtime_end") + if start: + rows = [r for r in rows if r["date"] >= start] + if end: + rows = [r for r in rows if r["date"] <= end] + # FRED only returns scheduled dates that have not produced data yet when this flag + # is set, so the fixture withholds them without it. + if params.get("include_release_dates_with_no_data") != "true": + rows = [r for r in rows if r["date"] <= now.isoformat()] + return 200, {"count": len(rows), "release_dates": rows[: int(params.get("limit", 1000))]} + + +# --- callers ------------------------------------------------------------------------- class MockFred: - """Routes FRED paths to captured payloads and records every request.""" + """httpx transport over ``route``, recording every request the tests assert on.""" - def __init__(self) -> None: + def __init__(self, today: date = FIXED_TODAY) -> None: + self.today = today self.requests: list[tuple[str, dict[str, str]]] = [] def transport(self) -> httpx.MockTransport: @@ -165,174 +424,44 @@ def query(self, path_suffix: str) -> dict[str, str]: raise AssertionError(f"no request recorded for {path_suffix}; saw {[p for p, _ in self.requests]}") def _handle(self, request: httpx.Request) -> httpx.Response: - path = request.url.path params = dict(request.url.params) - self.requests.append((path, params)) - - # Exact, not endswith: /release/series and /category/series also end in - # "/series" and would otherwise be swallowed by this branch. - if path.endswith("/fred/series"): - return self._one_series(params.get("series_id", "")) - if path.endswith("/series/observations"): - return self._observations(params) - if path.endswith("/series/vintagedates"): - return _ok({"count": len(VINTAGE_DATES), "vintage_dates": VINTAGE_DATES[: int(params.get("limit", 100))]}) - if path.endswith("/releases/dates"): - return self._release_dates(params, RELEASE_DATES) - if path.endswith("/release/dates"): - # FRED omits release_name here, unlike /releases/dates. - rows = [ - {"release_id": r["release_id"], "date": r["date"]} - for r in RELEASE_DATES - if str(r["release_id"]) == params.get("release_id") - ] - return self._release_dates(params, rows) - if path.endswith("/fred/release"): - return _ok({"releases": [{"id": 50, "name": "Employment Situation", "link": "http://www.bls.gov/ces/"}]}) - if path.endswith("/series/search"): - return self._series_list(params, [UNRATE, UNRATENSA, CPIAUCSL, GDPC1]) - if path.endswith("/release/series") or path.endswith("/category/series"): - return self._series_list(params, [UNRATE, CPIAUCSL]) - if path.endswith("/series/release"): - return _ok( - { - "releases": [ - { - "id": 50, - "realtime_start": "2026-08-05", - "realtime_end": "2026-08-05", - "name": "Employment Situation", - "press_release": True, - "link": "http://www.bls.gov/ces/", - } - ] - } - ) - if path.endswith("/series/categories"): - return _ok( - { - "categories": [ - { - "id": 32447, - "name": "Unemployment Rate", - "parent_id": 12, - "notes": "The ratio of unemployed to the civilian labor force.", - } - ] - } - ) - if path.endswith("/series/tags"): - return _ok( - { - "count": 2, - "tags": [ - {"name": "headline figure", "group_id": "gen", "notes": "", "popularity": 51}, - {"name": "monthly", "group_id": "freq", "notes": "", "popularity": 93}, - ], - } - ) - return _error(404, f"Not Found. No handler for {path}.") - - def _observations(self, params: dict[str, str]) -> httpx.Response: - series_id = params.get("series_id", "") - if series_id not in OBSERVATIONS: - return _error(400, "Bad Request. The series does not exist.") - - output_type = params.get("output_type", "1") - if output_type in ("2", "4"): - # FRED's own behaviour, and the reason get_revisions exists: without a - # real-time window spanning the record, a vintage request fails. - if params.get("realtime_start") != "1776-07-04": - return _error( - 400, - "Bad Request. No vintage dates exist for the specified real-time period: " - f"{TODAY} to {TODAY}.", - ) - if output_type == "2": - row = VINTAGE_ROW if params.get("observation_start") == VINTAGE_ROW["date"] else None - return _ok({"observations": [row] if row else []}) - rows = INITIAL_OBS.get(series_id, []) - return self._observation_payload(_limited(rows, params), params) - - rows = OBSERVATIONS[series_id] - start, end = params.get("observation_start"), params.get("observation_end") - if start: - rows = [r for r in rows if r[0] >= start] - if end: - rows = [r for r in rows if r[0] <= end] - return self._observation_payload(_limited(rows, params), params) - - def _observation_payload(self, rows: list[tuple[str, str]], params: dict[str, str]) -> httpx.Response: - # Real-time fields carry the same value on every row, which is the redundancy - # the columnar shaping exists to remove; the fixture reproduces it faithfully. - return _ok( - { - "realtime_start": TODAY, - "realtime_end": TODAY, - "units": params.get("units", "lin"), - "count": len(rows), - "observations": [ - {"realtime_start": TODAY, "realtime_end": TODAY, "date": d, "value": v} for d, v in rows - ], - } - ) - - def _one_series(self, series_id: str) -> httpx.Response: - if series_id not in SERIES: - return _error(400, "Bad Request. The series does not exist.") - return _ok({"realtime_start": "2026-08-05", "realtime_end": "2026-08-05", "seriess": [SERIES[series_id]]}) - - def _series_list(self, params: dict[str, str], pool: list[dict]) -> httpx.Response: - """Applies FRED's tag filter, its ordering, and its limit.""" - rows = list(pool) - for tag in filter(None, params.get("tag_names", "").split(";")): - rows = [r for r in rows if tag in _tags_for(r)] - - order_by = params.get("order_by", "series_id") - reverse = params.get("sort_order", "asc") == "desc" - if order_by in {"popularity", "group_popularity"}: - rows.sort(key=lambda r: r.get(order_by, 0), reverse=reverse) - else: - rows.sort(key=lambda r: str(r.get(order_by, r["id"])), reverse=reverse) - - total = len(rows) - limit = int(params.get("limit", 1000)) - return _ok({"count": total, "offset": 0, "limit": limit, "seriess": rows[:limit]}) - - - def _release_dates(self, params: dict[str, str], pool: list[dict]) -> httpx.Response: - rows = list(pool) - start, end = params.get("realtime_start"), params.get("realtime_end") - if start: - rows = [r for r in rows if r["date"] >= start] - if end: - rows = [r for r in rows if r["date"] <= end] - # FRED only returns scheduled dates that have not produced data yet when this - # flag is set, so the fixture withholds them without it. - if params.get("include_release_dates_with_no_data") != "true": - rows = [r for r in rows if r["date"] <= TODAY] - return _ok({"count": len(rows), "release_dates": rows[: int(params.get("limit", 1000))]}) - - -def _limited(rows: list[tuple[str, str]], params: dict[str, str]) -> list[tuple[str, str]]: - """FRED's limit and sort_order, applied the way the real API does.""" - if params.get("sort_order") == "desc": - rows = sorted(rows, reverse=True) - limit = int(params.get("limit", 100000)) - return rows[:limit] + self.requests.append((request.url.path, params)) + status, body = route(request.url.path, params, self.today) + return httpx.Response(status, json=body) -def _tags_for(series: dict) -> set[str]: - """The freq and seas tags FRED would carry for a series, derived from its fields.""" - return { - series["frequency"].lower(), - "nsa" if series["seasonal_adjustment_short"] == "NSA" else "sa", - } +def serve(port: int = 8080) -> None: + """Serve the mock over HTTP. This is what the benchmark container runs. + + stdlib only, on purpose: the plugin's own dependencies are mcp, httpx and pydantic, + and the benchmark should not be the reason a web framework joins them. + """ + from http.server import BaseHTTPRequestHandler, HTTPServer + from urllib.parse import parse_qs, urlparse + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 (stdlib's spelling) + parsed = urlparse(self.path) + params = {k: v[0] for k, v in parse_qs(parsed.query).items()} + status, body = route(parsed.path, params) + payload = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args: object) -> None: + pass # quiet: the container's log is for the agent, not for request lines + + HTTPServer(("0.0.0.0", port), Handler).serve_forever() + +if __name__ == "__main__": + import sys -def _ok(payload: dict) -> httpx.Response: - return httpx.Response(200, json=payload) + serve(int(sys.argv[1]) if len(sys.argv) > 1 else 8080) -def _error(status: int, message: str) -> httpx.Response: - return httpx.Response(status, json={"error_code": status, "error_message": message}) +# Kept for the tests that assert against the pinned clock. +TODAY = FIXED_TODAY.isoformat() diff --git a/plugins/fred/tests/integration/test_discovery.py b/plugins/fred/tests/integration/test_discovery.py index 508a4bb..13189b9 100644 --- a/plugins/fred/tests/integration/test_discovery.py +++ b/plugins/fred/tests/integration/test_discovery.py @@ -17,7 +17,7 @@ async def test_free_text_search_orders_by_popularity(self, call, fred): async def test_the_key_is_sent_but_the_count_is_freds(self, call, fred): out = await call("search_series", query="unemployment") - assert out["count"] == 4 + assert out["count"] == 5 # the whole search pool assert out["returned"] == len(out["series"]) async def test_release_id_switches_endpoint_without_changing_the_shape(self, call, fred): diff --git a/plugins/fred/tests/integration/test_observations.py b/plugins/fred/tests/integration/test_observations.py index 857dd98..96b653d 100644 --- a/plugins/fred/tests/integration/test_observations.py +++ b/plugins/fred/tests/integration/test_observations.py @@ -111,6 +111,18 @@ async def test_the_summary_still_reports_the_true_extremes(self, call): assert summary["count"] == len(DAILY_OBS) assert summary["observations"] == len(DAILY_OBS) + async def test_the_extremes_are_not_in_the_returned_points(self, call): + """The task `rate-history-max` is only a test if this holds. + + The summary must be the *only* route to the true extremes. An earlier fixture + used a broad peak that the sampling grid landed on exactly, so an agent reading + the returned points got the right answer and the eval proved nothing. + """ + out = await call("get_observations", series_ids="DGS10") + sampled = [v for v in out["values"]["DGS10"] if v is not None] + assert max(sampled) < DAILY_MAX + assert min(sampled) > DAILY_MIN + async def test_the_first_and_last_dates_survive(self, call): out = await call("get_observations", series_ids="DGS10") assert out["dates"][0] == DAILY_OBS[0][0] diff --git a/plugins/fred/tests/integration/test_revisions_and_calendar.py b/plugins/fred/tests/integration/test_revisions_and_calendar.py index a3c6486..4d382a5 100644 --- a/plugins/fred/tests/integration/test_revisions_and_calendar.py +++ b/plugins/fred/tests/integration/test_revisions_and_calendar.py @@ -1,12 +1,17 @@ """get_revisions and get_release_calendar, through the registered MCP server.""" -from datetime import date +from datetime import date, timedelta import pytest from src import dates as dates_module -from ..fixtures.fred_api import TODAY +from ..fixtures.fred_api import FIXED_TODAY, NEXT_RELEASE_50_OFFSET, RELEASE_OFFSETS, TODAY + + +def day(offset: int) -> str: + """A fixture release date, as the calendar will report it.""" + return (FIXED_TODAY + timedelta(days=offset)).isoformat() pytestmark = pytest.mark.integration @@ -101,8 +106,10 @@ class TestReleaseCalendar: async def test_splits_on_today(self, call): out = await call("get_release_calendar") assert out["today"] == TODAY - assert [r["date"] for r in out["released"]] == ["2026-08-01", "2026-08-05"] - assert [r["date"] for r in out["upcoming"]] == ["2026-08-07", "2026-08-12"] + # Offsets, not literals: the fixture generates release dates relative to the + # clock, so a hard-coded date would only be right for one day. + assert [r["date"] for r in out["released"]] == [day(-5), day(-2), day(0)] + assert [r["date"] for r in out["upcoming"]] == [day(9), day(12)] async def test_a_release_dated_today_counts_as_released(self, call): # It has come out; a model asking "what came out today" should see it. @@ -142,7 +149,18 @@ async def test_a_limited_page_says_how_much_it_left_out(self, call): out = await call("get_release_calendar", limit=1) assert len(out["released"]) == 1 assert len(out["upcoming"]) == 1 - assert out["totals"] == {"released": 2, "upcoming": 2} + assert out["totals"] == {"released": 3, "upcoming": 2} + + async def test_the_next_release_for_one_publication(self, call): + # What `next-release` in the eval suite asks for, on the default window. + out = await call("get_release_calendar", release_id=50) + assert out["upcoming"][0]["date"] == day(NEXT_RELEASE_50_OFFSET) + + async def test_a_wide_window_reaches_every_date_for_a_release(self, call): + # `end` takes absolute dates as well as spans. Spans only run backwards + # ("5y" is five years ago), so a forward window is written out in full. + out = await call("get_release_calendar", release_id=50, start="2020-01-01", end="2030-01-01") + assert len(out["released"]) + len(out["upcoming"]) == len(RELEASE_OFFSETS[50]) async def test_release_id_narrows_to_one_publication_and_names_it(self, call, fred): out = await call("get_release_calendar", release_id=50) diff --git a/plugins/fred/tests/unit/test_client.py b/plugins/fred/tests/unit/test_client.py index 36d7e72..c2f78c3 100644 --- a/plugins/fred/tests/unit/test_client.py +++ b/plugins/fred/tests/unit/test_client.py @@ -143,3 +143,23 @@ async def test_a_missing_key_surfaces_at_call_time_not_construction(self, monkey client = FredClient(transport=httpx.MockTransport(lambda r: httpx.Response(200, json={}))) with pytest.raises(CredentialsError): await client.get("/series") + + +class TestBaseUrl: + def test_defaults_to_the_real_api(self): + assert FredClient().base_url == "https://api.stlouisfed.org/fred" + + def test_the_environment_can_point_it_at_a_mock(self, monkeypatch): + # This is what lets the eval benchmark run against a local mock with no key + # and no possibility of reaching the real API. .env.example documented it + # before the code read it. + monkeypatch.setenv("FRED_BASE_URL", "http://localhost:8080/fred") + assert FredClient().base_url == "http://localhost:8080/fred" + + def test_an_explicit_argument_beats_the_environment(self, monkeypatch): + monkeypatch.setenv("FRED_BASE_URL", "http://localhost:8080/fred") + assert FredClient(base_url="http://other:9/fred").base_url == "http://other:9/fred" + + def test_a_trailing_slash_does_not_double_up(self, monkeypatch): + monkeypatch.setenv("FRED_BASE_URL", "http://localhost:8080/fred/ ") + assert FredClient().base_url == "http://localhost:8080/fred" diff --git a/plugins/fred/tests/unit/test_server.py b/plugins/fred/tests/unit/test_server.py index 0ddedf6..ccbed0a 100644 --- a/plugins/fred/tests/unit/test_server.py +++ b/plugins/fred/tests/unit/test_server.py @@ -47,9 +47,14 @@ def test_the_launcher_is_executable(self): # anyone running it directly gets a permission error. assert (ROOT / "scripts" / "start-server.sh").stat().st_mode & 0o111 - def test_the_api_key_is_passed_through(self): + def test_every_setting_the_server_reads_is_passed_through(self): + """An env var the code honours but .mcp.json drops does nothing once installed. + + FRED_BASE_URL was documented in .env.example, unread by the code, and absent + here; two of those three were fixed together, so this pins the third. + """ config = json.loads((ROOT / ".mcp.json").read_text()) - assert "FRED_API_KEY" in config["mcpServers"]["fred"]["env"] + assert set(config["mcpServers"]["fred"]["env"]) == {"FRED_API_KEY", "FRED_BASE_URL", "FRED_LOG_LEVEL"} def test_plugin_name_matches_the_server_name(self): manifest = json.loads((ROOT / ".claude-plugin" / "plugin.json").read_text())