diff --git a/.github/workflows/tastytrade.yml b/.github/workflows/tastytrade.yml index 1be0c8d..9bad28e 100644 --- a/.github/workflows/tastytrade.yml +++ b/.github/workflows/tastytrade.yml @@ -43,3 +43,102 @@ jobs: - name: Validate eval task verifiers run: make validate-tasks + + evals: + # THE MERGE GATE on the tool surface and the earnings-calendars skill: drives + # every task with the real claude-code agent against the mock API, and fails + # unless the rewards clear the threshold. + # + # `check` above runs make validate-tasks, which is a different question: it + # proves each verifier accepts its oracle and rejects an empty answer, with no + # model involved. That catches a broken verifier. Only this catches a server or + # skill that a real agent cannot actually drive. + # + # Unlike every other step here this one spends model tokens, so it needs `check` + # to pass first and is guarded to same-repo PRs (a fork PR gets no secrets). + # + # It spends them against a Claude subscription rather than API credits. + # CLAUDE_FORCE_OAUTH makes harbor's claude-code agent blank ANTHROPIC_API_KEY + # before building the container env, so only CLAUDE_CODE_OAUTH_TOKEN survives + # and the CLI falls back to subscription auth. Never add ANTHROPIC_API_KEY back: + # with both present and the flag unset the CLI silently prefers the key and the + # run is on credits again, with only a debug log to say so. + # + # Serialized on purpose. Subscription rate limits are per account and shared + # with interactive use. Note this group does not serialize against harbor-hub's + # gate, which draws on the same account; if the two start colliding, give both + # workflows one shared group name. + runs-on: ubuntu-latest + needs: [check] + if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }} + concurrency: + group: tastytrade-evals + cancel-in-progress: false + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_FORCE_OAUTH: "1" + # For --upload on pushes to main. Results land on the hub as + # `ci-evals-tastytrade`, following the repo-wide `ci-evals-` + # convention so every plugin's CI history is searchable together. + HARBOR_API_KEY: ${{ secrets.HARBOR_API_KEY }} + # One attempt per task. 13 tasks is already the bulk of a gate run, and a + # task that only passes sometimes is a signal to fix the task, not to + # average it away. + EVAL_ATTEMPTS: "1" + EVAL_MIN_MEAN: "1.0" + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.13" + + # A setup-token credential is long-lived but not permanent. Without this + # probe an expired one surfaces as "the gate did not clear 1.0", which reads + # as a server regression and isn't. Fails rather than skips on a missing + # token: a secret-dependent job silently going green is the worse failure. + - name: Check the subscription token + run: | + set -euo pipefail + if [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then + echo "::error::CLAUDE_CODE_OAUTH_TOKEN is not set. Mint one with 'claude setup-token' and add it as a repository secret." + exit 1 + fi + status="$(curl -sS -o /tmp/auth-probe.json -w '%{http_code}' \ + https://api.anthropic.com/v1/messages \ + -H "Authorization: Bearer $CLAUDE_CODE_OAUTH_TOKEN" \ + -H "anthropic-version: 2023-06-01" \ + -H "anthropic-beta: oauth-2025-04-20" \ + -H "content-type: application/json" \ + -d '{"model":"claude-haiku-4-5","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}')" + if [ "$status" = "401" ]; then + echo "::error::CLAUDE_CODE_OAUTH_TOKEN is expired or revoked. Mint a new one with 'claude setup-token' and update the repository secret." + exit 1 + fi + if [ "$status" != "200" ]; then + echo "::error::Auth probe returned HTTP $status, expected 200. The gate would likely fail after spending a rate-limit window; response body:" + cat /tmp/auth-probe.json + exit 1 + fi + echo "Auth probe returned HTTP 200; proceeding." + + - name: Run the eval gate (claude-code) + env: + EVALS_OUT_DIR: ${{ runner.temp }}/eval-trials + # Every gate run uploads, PR included. Gating this to pushes on main + # meant a PR gate left nothing on the hub, so the only record of a run + # was a CI artifact that expires in 7 days. One job per run is cheap; + # a result you cannot find later is not. + EVALS_UPLOAD: "1" + run: make evals + + # always(), not failure(): harbor draws progress as a live TUI, so a run + # that passes slowly logs almost nothing. The trials are the only record a + # PR run produces, and a failure is only diagnosable from the trajectory. + - name: Upload trials for diagnosis + if: always() + uses: actions/upload-artifact@v4 + with: + name: tastytrade-eval-trials + path: ${{ runner.temp }}/eval-trials + retention-days: 7 diff --git a/plugins/harbor-hub/evals/run_evals.sh b/plugins/harbor-hub/evals/run_evals.sh index 36caf72..03d5703 100755 --- a/plugins/harbor-hub/evals/run_evals.sh +++ b/plugins/harbor-hub/evals/run_evals.sh @@ -32,6 +32,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" HARBOR_TEST_ENV="${HARBOR_TEST_ENV:-docker}" +# Repo-wide convention: ci-evals-, so every plugin's CI runs are +# searchable together on the hub instead of hiding behind a generic name. +JOB_NAME="ci-evals-harbor-hub" EVAL_TASK_REF="${EVAL_TASK_REF:-hello-world/hello-world@1}" # Git ref the eval images pip-install harbor-mcp from. The default branch is # only right for a run of main: as a PR gate the images must be built from the @@ -115,7 +118,7 @@ run_args=( -a claude-code -e "$HARBOR_TEST_ENV" -o "$JOBS_DIR" - --job-name evals-gate + --job-name "$JOB_NAME" --ae HARBOR_API_KEY="$HARBOR_API_KEY" --ae EVAL_READ_JOB_ID="$READ_JOB_ID" --ae EVAL_DELETE_JOB_ID="$DELETE_JOB_ID" @@ -135,9 +138,9 @@ harbor run "${run_args[@]}" # harbor run exits 0 regardless of reward; gate on a perfect result so CI # (and `make evals`) fails the moment any eval regresses. -if ! python3 "$REPO_ROOT/evals/check_reward.py" "$JOBS_DIR/evals-gate/result.json" evals-gate; then +if ! python3 "$REPO_ROOT/evals/check_reward.py" "$JOBS_DIR/$JOB_NAME/result.json" "$JOB_NAME"; then echo "--- verifier output ---" >&2 - cat "$JOBS_DIR/evals-gate"/*/verifier/test-stdout.txt >&2 2>/dev/null || true + cat "$JOBS_DIR/$JOB_NAME"/*/verifier/test-stdout.txt >&2 2>/dev/null || true die "the evals did not all reach reward 1.0" fi diff --git a/plugins/tastytrade/.claude-plugin/plugin.json b/plugins/tastytrade/.claude-plugin/plugin.json index 1e8a1ad..fd1183d 100644 --- a/plugins/tastytrade/.claude-plugin/plugin.json +++ b/plugins/tastytrade/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "tastytrade", - "version": "0.4.0", + "version": "0.4.1", "description": "MCP server for the TastyTrade Open API: brokerage accounts, positions, market data, option chains, transactions, and order preview.", "author": { "name": "Walker Hughes" diff --git a/plugins/tastytrade/.dockerignore b/plugins/tastytrade/.dockerignore new file mode 100644 index 0000000..31c686f --- /dev/null +++ b/plugins/tastytrade/.dockerignore @@ -0,0 +1,15 @@ +# The benchmark image copies this directory, so keep out anything large, local, or +# secret. .env especially: the image ships throwaway credentials on purpose and must +# never pick up real ones. +.env +.venv +.git +.githooks +evals/jobs +__pycache__ +*.pyc +.pytest_cache +.ruff_cache +.mypy_cache +.coverage +htmlcov diff --git a/plugins/tastytrade/Makefile b/plugins/tastytrade/Makefile index ae656d8..48fddca 100644 --- a/plugins/tastytrade/Makefile +++ b/plugins/tastytrade/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint lint-fix format typecheck check selftest test test-unit test-integration coverage install-hooks validate-tasks mock-api benchmark-build benchmark benchmark-view +.PHONY: lint lint-fix format typecheck check selftest evals test test-unit test-integration coverage install-hooks validate-tasks mock-api benchmark-build benchmark benchmark-view lint: uv run ruff check . @@ -38,6 +38,13 @@ install-hooks: validate-tasks: uv run python evals/generate_tasks.py bash evals/validate_local.sh + python3 evals/check_reward.py --selftest + +# THE MERGE GATE: drives the tasks with a real agent. Spends model tokens, needs +# Docker and CLAUDE_CODE_OAUTH_TOKEN. validate-tasks proves the verifiers work; +# only this proves an agent can actually drive the server and the skill. +evals: + bash evals/run_gate.sh # Run the mock Tastytrade API on its own, the way the eval benchmark runs it. mock-api: @@ -45,12 +52,14 @@ mock-api: # Benchmark targets (need Docker running and ANTHROPIC_API_KEY set). These cd into evals/ so # Harbor resolves the task path correctly regardless of where you invoke make, and call Harbor -# through uv so it does not need to be on PATH. The version is pinned to the one the tasks were -# validated against. Override with e.g. HARBOR=harbor to use a different harbor. -HARBOR ?= uv tool run --from "harbor==0.13.2" harbor +# through uv so it does not need to be on PATH. Pinned to 0.18.0, matching harbor-hub: 0.13.2's +# --upload wanted a `harbor auth login` credentials file and ignored HARBOR_API_KEY, so the CI +# gate could not upload. Override with e.g. HARBOR=harbor to use a different harbor. +HARBOR ?= uv tool run --from "harbor==0.18.0" harbor +# Context is the plugin root so the image can copy the working tree it is testing. benchmark-build: - docker build -t tastytrade-bench evals/environment + docker build -f evals/environment/Dockerfile -t tastytrade-bench . benchmark: cd evals && $(HARBOR) run -c job.yaml diff --git a/plugins/tastytrade/evals/README.md b/plugins/tastytrade/evals/README.md index c5f477e..f6fcc77 100644 --- a/plugins/tastytrade/evals/README.md +++ b/plugins/tastytrade/evals/README.md @@ -1,10 +1,13 @@ # Evals -The server is evaluated at the agent-loop level. Claude Code drives the tools over a set of -tasks, once against the baseline server (`main`) and once against the candidate server -(`mcp-server-refactor`), using the [Harbor](https://github.com/laude-institute/harbor) -framework. The agent is the same in both runs and only the MCP server changes, so any -difference in success rate, tool calls, tokens, or latency comes from the server itself. +The server is evaluated at the agent-loop level: Claude Code drives the tools over a set of +tasks using the [Harbor](https://github.com/laude-institute/harbor) framework, and each trial +records reward, tool calls, tokens, and latency. + +This used to run two agents, a baseline ref and a candidate ref, to show the refactored server +beat the plain endpoint-wrapper baseline. That refactor landed on main and the candidate branch +was deleted, which left the image cloning a ref that no longer existed. There is no second side +to compare against now, so the job runs one server and the evals serve as a regression gate. Every task runs against the mock Tastytrade API in `tests/fixtures/mock_api`, so the answers are fixed and reproducible and no run touches a real account or live market data. The fast @@ -16,30 +19,39 @@ in `tests/unit/test_misuse_evals.py`. ``` evals/ environment/ - Dockerfile # python and uv, both server checkouts, the mock API - scripts/ # require-local-api, start-mock, mcp-baseline, mcp-candidate + Dockerfile # python, uv, the server checkout, the skill, the mock API + scripts/ # require-local-api, start-mock, mcp-server tasks// task.toml # task config instruction.md # the prompt the agent sees tests/test.sh # verifier, writes a reward to /logs/verifier/reward.txt solution/solve.sh # oracle, writes the known-correct answer - job.yaml # runs the baseline and candidate agents over every task + job.yaml # runs the agent over every task generate_tasks.py # regenerates the tasks from the fixtures validate_local.sh # checks every verifier without Harbor or Docker ``` -## Tasks (12) +## Tasks (13) Ten tasks ask for a single number (portfolio P/L, largest drawdown, ATM strike, IV rank, total fees, net cash, latest dividend, net liquidating value, position count, and the fees on -a previewed vertical spread). One asks for the symbols in a watchlist. The last asks the agent +a previewed vertical spread). One asks for the symbols in a watchlist. One asks the agent to place an order, and its verifier reads the order the mock recorded rather than a file the agent wrote. +The last, `earnings-implied-move`, is the only one that exercises a skill rather than the MCP +tools. It points the agent at the option chain that `earnings-calendars` ships and asks for the +implied expected earnings move, which is 10.52% for that chain. The prompt names neither the +skill nor the script, so it measures whether the agent recognises an earnings-vol question and +reaches for the right tool. The image installs the skill at `/root/.claude/skills` so the normal +trigger path is live. + Nothing in the tasks is hand-typed. `generate_tasks.py` computes every expected answer from the mock fixtures by running the same shaping code the server uses, so a task can never -disagree with the data the agent sees. The two anchor values are a total unrealized P/L of -+$700 and an SPY ATM strike of 200. Regenerate after changing a fixture: +disagree with the data the agent sees. The skill task follows the same rule: its answer comes +from importing `scripts/calendars.py` and fitting the shipped chain. The two anchor values are +a total unrealized P/L of +$700 and an SPY ATM strike of 200. Regenerate after changing a +fixture: ```bash python evals/generate_tasks.py @@ -53,7 +65,7 @@ repo root: ```bash export ANTHROPIC_API_KEY=... -make benchmark-build # docker build -t tastytrade-bench evals/environment +make benchmark-build # docker build -f evals/environment/Dockerfile -t tastytrade-bench . make benchmark # cd evals && harbor run -c job.yaml make benchmark-view # cd evals && harbor view jobs ``` @@ -62,10 +74,10 @@ Or run Harbor directly, from inside `evals/`. Call it through `uv` and pin the v tasks were validated against, so it doesn't depend on what's on your PATH: ```bash +docker build -f evals/environment/Dockerfile -t tastytrade-bench . cd evals -docker build -t tastytrade-bench environment -uv tool run --from "harbor==0.13.2" harbor run -c job.yaml -uv tool run --from "harbor==0.13.2" harbor view jobs +uv tool run --from "harbor==0.18.0" harbor run -c job.yaml +uv tool run --from "harbor==0.18.0" harbor view jobs ``` Each task carries a one-line `environment/Dockerfile` (`FROM tastytrade-bench`). Harbor only @@ -75,8 +87,37 @@ they inherit. The tasks reference the image by name (`docker_image = "tastytrade-bench"`), so build it before the first run. Each trial's `result.json` records the reward, the phase timings, and -the token and cost totals, so success rate, tokens, tool calls, and latency per server come -straight out of the job directory. +the token and cost totals, so success rate, tokens, tool calls, and latency come straight out +of the job directory. + +The image copies the working tree rather than cloning a ref, so a run measures the code you +have checked out. Rebuild after changing the server or the skill. + +## The merge gate + +`make validate-tasks` and `make evals` answer different questions, and CI runs both. +`validate-tasks` proves each verifier accepts its oracle and rejects an empty answer, with no +model involved, which catches a broken verifier. `make evals` drives all 13 tasks with the +real claude-code agent and fails unless every reward is 1.0, which is the only thing that +catches a server or skill an agent cannot actually drive. + +The gate authenticates with `CLAUDE_CODE_OAUTH_TOKEN` so runs bill to a Claude subscription +rather than API credits. It deliberately does **not** accept `ANTHROPIC_API_KEY`: with a key +present the CLI prefers it over the token, which either moves the run onto credits silently or, +if the key is empty, 401s every trial before spending a token. `job.yaml` used to declare the +key for exactly this reason and no longer does. + +Every gate run uploads to the Harbor hub as **`ci-evals-tastytrade`**, one job per CI run +holding all 13 tasks as trials. That follows the repo-wide `ci-evals-` convention, so +every plugin's CI history is searchable together instead of hiding behind a generic job name. + +PR runs upload too. Restricting uploads to main left a PR gate with nothing on the hub, so the +only record was a CI artifact that expires after 7 days. + +```bash +export CLAUDE_CODE_OAUTH_TOKEN=... # claude setup-token +make evals # add HARBOR_API_KEY and EVALS_UPLOAD=1 to upload +``` ## Check the verifiers without Harbor @@ -86,7 +127,7 @@ reward is 0. This is the local stand-in for `harbor run -a oracle`: ```bash bash evals/validate_local.sh -# 12 passed, 0 failed +# 13 passed, 0 failed ``` ## Safety diff --git a/plugins/tastytrade/evals/check_reward.py b/plugins/tastytrade/evals/check_reward.py new file mode 100644 index 0000000..72e2e7a --- /dev/null +++ b/plugins/tastytrade/evals/check_reward.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Gate a harbor ``result.json`` on its rewards. + +``harbor run`` exits 0 whatever the reward, so the gate has to inspect the job +result itself. Every tastytrade task reports one reward in [0, 1], so the mean +over a task is its pass rate across attempts. + + python3 check_reward.py # every reward 1.0 + python3 check_reward.py --min-mean 0.9 # allow some slack + python3 check_reward.py --selftest + +``--min-mean`` exists because this gate drives a real agent over 13 tasks, not +3. A single flaky task should not be indistinguishable from a broken server, +but the threshold is an explicit number in the workflow rather than a silent +default, so loosening it is a visible decision. +""" + +import json +import sys +from pathlib import Path + + +def rewards(stats: dict) -> tuple[list | None, str]: + """Every reward in the run, or (None, reason) if it did not complete cleanly.""" + if stats.get("n_errored_trials") or not stats.get("n_completed_trials"): + return None, f"run did not complete cleanly (stats={stats})" + found = [ + value + for eval_stats in stats.get("evals", {}).values() + for metric in eval_stats.get("metrics", []) + for value in metric.values() + ] + if not found: + return None, "no rewards reported" + return found, "" + + +def gate(stats: dict, min_mean: float = 1.0) -> tuple[bool, str]: + found, reason = rewards(stats) + if found is None: + return False, reason + mean = sum(found) / len(found) + failed = [r for r in found if r < min_mean] + n = stats["n_completed_trials"] + if min_mean >= 1.0 and failed: + return False, f"reward not perfect (mean={mean:.3f}, rewards={found})" + if mean < min_mean: + return False, f"mean reward {mean:.3f} below threshold {min_mean} (rewards={found})" + return True, f"mean reward {mean:.3f} over {n} trial(s), {len(found)} task(s)" + + +def _selftest() -> None: + ok = {"n_completed_trials": 2, "evals": {"a": {"metrics": [{"mean": 1.0}]}}} + assert gate(ok)[0] + assert not gate({**ok, "evals": {"a": {"metrics": [{"mean": 0.0}]}}})[0] + assert not gate({**ok, "n_errored_trials": 1})[0] + assert not gate({"n_completed_trials": 0, "evals": {}})[0] + assert not gate({"n_completed_trials": 1, "evals": {}})[0] + + mixed = { + "n_completed_trials": 2, + "evals": {"a": {"metrics": [{"mean": 1.0}]}, "b": {"metrics": [{"mean": 0.0}]}}, + } + assert not gate(mixed)[0], "one zero must fail a perfect gate" + assert not gate(mixed, min_mean=0.9)[0], "mean 0.5 is below 0.9" + assert gate(mixed, min_mean=0.5)[0], "mean 0.5 meets a 0.5 threshold" + print("check_reward selftest ok") + + +def main(argv: list[str]) -> int: + if argv[1:2] == ["--selftest"]: + _selftest() + return 0 + result_path, name = argv[1], argv[2] + min_mean = 1.0 + if "--min-mean" in argv: + min_mean = float(argv[argv.index("--min-mean") + 1]) + try: + stats = json.loads(Path(result_path).read_text()).get("stats", {}) + except FileNotFoundError: + print(f"{name}: no result.json at {result_path}", file=sys.stderr) + return 1 + ok, msg = gate(stats, min_mean) + print(f"{name}: {msg}", file=sys.stdout if ok else sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/plugins/tastytrade/evals/environment/Dockerfile b/plugins/tastytrade/evals/environment/Dockerfile index 2554478..a7f92e2 100644 --- a/plugins/tastytrade/evals/environment/Dockerfile +++ b/plugins/tastytrade/evals/environment/Dockerfile @@ -1,8 +1,17 @@ # Benchmark agent environment. # -# Provides two checkouts of the server, the baseline (main) and the candidate -# (mcp-server-refactor), plus the mock Tastytrade API, so Claude Code can be benchmarked -# against each one over the same tasks. Harbor installs the claude-code agent itself. +# One checkout of the server plus the mock Tastytrade API, so Claude Code can be +# benchmarked over the eval tasks. Harbor installs the claude-code agent itself. +# +# This used to build two checkouts and compare a baseline ref against a candidate +# ref. That comparison existed to show the refactored server beat the plain +# endpoint-wrapper baseline. The refactor landed on main, the candidate branch was +# deleted, and the image stopped building. There is no second side left to compare +# against, so the split is gone and the evals are now a regression gate on one +# server rather than a bake-off between two. +# +# Build context is the plugin root, not this directory: +# make benchmark-build # docker build -f evals/environment/Dockerfile . FROM python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends git curl ca-certificates \ @@ -11,35 +20,25 @@ RUN apt-get update && apt-get install -y --no-install-recommends git curl ca-cer # uv for dependency management (matches the project toolchain). COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -# BROKEN, needs a decision before `make benchmark` can build again. -# -# This image compares a baseline ref against a candidate ref. CANDIDATE_REF names -# `mcp-server-refactor`, a branch that no longer exists, so the clone below fails. -# The old default REPO also pointed at walkerhughes/tastytrade-mcp, which was -# consolidated into this monorepo and last received a commit in June. -# -# The repo and subdirectory are corrected here, but the two-ref comparison itself -# no longer has a second ref to compare against. Pick one: -# - point CANDIDATE_REF at a tag or commit worth benchmarking against main, or -# - drop the baseline/candidate split and benchmark a single checkout. -# Nothing in CI exercises this path (`make validate-tasks` does not build the -# image), which is why it went unnoticed. -ARG REPO=https://github.com/walkerhughes/claude -ARG BASELINE_REF=main -ARG CANDIDATE_REF=mcp-server-refactor +# The server under test is the working tree rather than a cloned ref. A clone only +# ever measured what had already been pushed, which is how the stale-branch breakage +# went unnoticed; copying measures the code actually in front of you. +COPY . /opt/tastytrade +RUN cd /opt/tastytrade && (uv sync --frozen || uv sync) -# Both checkouts. The mock API and fixtures live in the candidate checkout and serve both. -RUN git clone --depth 1 --branch ${BASELINE_REF} ${REPO} /opt/mcp-baseline \ - && git clone --depth 1 --branch ${CANDIDATE_REF} ${REPO} /opt/mcp-candidate -RUN cd /opt/mcp-baseline/plugins/tastytrade && uv sync --frozen || uv sync -RUN cd /opt/mcp-candidate/plugins/tastytrade && uv sync --frozen || uv sync +# Claude Code discovers personal skills here. Installing the skill lets a task +# exercise the real trigger path instead of naming the script in its prompt. +# Every task image is FROM this one, so the agent container gets it too. Harbor +# also has a --skills flag that mounts a directory; this stays in the image +# because it can be verified with a plain `docker run`, and that flag cannot. +RUN mkdir -p /root/.claude/skills \ + && cp -r /opt/tastytrade/skills/earnings-calendars /root/.claude/skills/earnings-calendars -COPY scripts/ /usr/local/bin/ -RUN chmod +x /usr/local/bin/require-local-api /usr/local/bin/start-mock \ - /usr/local/bin/mcp-baseline /usr/local/bin/mcp-candidate +COPY evals/environment/scripts/ /usr/local/bin/ +RUN chmod +x /usr/local/bin/require-local-api /usr/local/bin/start-mock /usr/local/bin/mcp-server WORKDIR /app -# Both servers talk to the mock API with throwaway credentials. require-local-api also +# The server talks to the mock API with throwaway credentials. require-local-api also # enforces the localhost target at startup, so the benchmark can never reach a real account. ENV API_BASE_URL=http://localhost:8080 \ TT_CLIENT_ID=test \ diff --git a/plugins/tastytrade/evals/environment/scripts/mcp-baseline b/plugins/tastytrade/evals/environment/scripts/mcp-baseline deleted file mode 100755 index fb86fe4..0000000 --- a/plugins/tastytrade/evals/environment/scripts/mcp-baseline +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -# Launch the baseline (main-branch) server over stdio against the mock API. -set -euo pipefail -require-local-api -start-mock -exec uv run --project /opt/mcp-baseline python -m src.server diff --git a/plugins/tastytrade/evals/environment/scripts/mcp-candidate b/plugins/tastytrade/evals/environment/scripts/mcp-candidate deleted file mode 100755 index d0bb770..0000000 --- a/plugins/tastytrade/evals/environment/scripts/mcp-candidate +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -# Launch the candidate (mcp-server-refactor) server over stdio against the mock API. -set -euo pipefail -require-local-api -start-mock -exec uv run --project /opt/mcp-candidate python -m src.server diff --git a/plugins/tastytrade/evals/environment/scripts/mcp-server b/plugins/tastytrade/evals/environment/scripts/mcp-server new file mode 100755 index 0000000..eb6e303 --- /dev/null +++ b/plugins/tastytrade/evals/environment/scripts/mcp-server @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Launch the server over stdio against the mock API. +# +# cd rather than `uv run --project`: the image's WORKDIR is /app so the agent writes its +# answer there, and `src` is only importable from the project root. --project alone sets +# the environment without moving the working directory, which is why this failed with +# ModuleNotFoundError until the image was actually built and run. +set -euo pipefail +require-local-api +start-mock +cd /opt/tastytrade +exec uv run python -m src.server diff --git a/plugins/tastytrade/evals/environment/scripts/start-mock b/plugins/tastytrade/evals/environment/scripts/start-mock index 642fd8b..fbd34dc 100755 --- a/plugins/tastytrade/evals/environment/scripts/start-mock +++ b/plugins/tastytrade/evals/environment/scripts/start-mock @@ -7,7 +7,7 @@ if curl -sf http://localhost:8080/oauth/token -X POST >/dev/null 2>&1; then exit 0 fi -cd /opt/mcp-candidate +cd /opt/tastytrade nohup uv run uvicorn tests.fixtures.mock_api.app:app --host 0.0.0.0 --port 8080 \ >/tmp/mock-api.log 2>&1 & diff --git a/plugins/tastytrade/evals/generate_tasks.py b/plugins/tastytrade/evals/generate_tasks.py index e61eb04..5d77a30 100644 --- a/plugins/tastytrade/evals/generate_tasks.py +++ b/plugins/tastytrade/evals/generate_tasks.py @@ -8,6 +8,7 @@ Run: python evals/generate_tasks.py """ +import importlib.util import json import os import stat @@ -55,6 +56,27 @@ def _watchlist_symbols(): return [e["symbol"] for e in entries] +# The reference chain the earnings-calendars skill ships. Loading the skill's own module +# keeps the same invariant as the fixture tasks: the expected answer is computed by the +# code under test, so it cannot drift away from what the agent will see. +SKILL_CHAIN = os.path.join(ROOT, "skills", "earnings-calendars", "reference", "pltr-2026-08-03.json") + + +def _calendars_module(): + spec = importlib.util.spec_from_file_location("calendars", os.path.join(ROOT, "scripts", "calendars.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _implied_move_pct(): + calendars = _calendars_module() + with open(SKILL_CHAIN) as fh: + chain = calendars.Chain(json.load(fh)) + _, _, jump = calendars.fit_term_structure(chain) + return round(calendars.expected_abs_move(jump) * 100, 2) + + # Each numeric task: directory name, prompt, the JSON key the agent must write, the value # (pulled from the fixtures above), and the tolerance the verifier allows. NUMERIC_TASKS = [ @@ -201,6 +223,26 @@ def _watchlist_symbols(): echo '{{"{key}": {expected}}}' > "$APP_DIR/answer.json" """ +# The skill task deliberately does not name the script or the skill. The point is whether +# the agent recognises an earnings-vol question and reaches for the right tool on its own, +# so naming it would test only that the agent can copy a command out of a prompt. +SKILL_INSTRUCTION = """\ +# Task: {title} + +PLTR reports earnings after the close today. A snapshot of its option chain, taken that +afternoon, is saved at: + + {chain} + +{instruction} + +Write it to `/app/answer.json` as a single JSON object with this shape, and nothing else: + +```json +{{"{key}": }} +``` +""" + CSV_INSTRUCTION = """\ # Task: {title} @@ -247,12 +289,26 @@ def _watchlist_symbols(): echo '{json_line}' > "$APP_DIR/answer.json" """ +# The sandbox sentence is load-bearing and true. Without it the first gate run +# stalled here: the agent read "place a market order" as a real financial +# transaction and stopped to ask for confirmation, which is the right instinct +# and the wrong outcome for a single-turn eval. Stating the environment is a +# mock removes a false safety concern rather than talking the agent past a real +# one, and the task still exercises the same place_order path. ORDER_INSTRUCTION = """\ # Task: Place an Order +This is an automated test against a mock brokerage API. There is no real account, no real +money, and no real market: orders are recorded to a local file and go nowhere else. Treat +this as a sandbox and do not stop to ask for confirmation. + Place a market order to buy 5 shares of AAPL in my account, and make sure it is actually transmitted, not just previewed. Trading is enabled on this server. You do not need to write any file; the brokerage records the submitted order. + +Use the Tastytrade MCP tools. Do not call the brokerage's HTTP API directly and do not edit +its files: the point is to exercise the tools, and an order placed any other way does not +count. """ # The verifier reads the order the mock recorded, so it checks the order the agent really @@ -367,6 +423,45 @@ def generate() -> list[str]: ) names.append(name) + # Earnings-calendar skill: the one task that exercises a skill rather than the MCP + # tools. The chain is a file in the image, so the answer is fixed and no market data + # is involved. + name = "earnings-implied-move" + base = os.path.join(TASKS_DIR, name) + key = "implied_expected_move_pct" + expected = _implied_move_pct() + # "Implied move" alone has two defensible readings, and the first gate run + # answered the other one: the agent priced the front straddle and reported + # 11.34% against an expected 10.52%. The front expiry carries four days of + # ordinary volatility on top of the event, so the straddle overstates the + # event itself. Saying that isolates the question without naming the skill, + # which is still the thing being measured. + instruction = ( + "From that chain, find the implied expected absolute move for the earnings event " + "itself, as a percent of the spot price. Isolate the event: the front expiry also " + "carries ordinary day-to-day volatility, and that part is not the answer." + ) + chain_in_image = "/opt/tastytrade/skills/earnings-calendars/reference/pltr-2026-08-03.json" + _write( + os.path.join(base, "task.toml"), + TASK_TOML.format(name=name, desc="Find PLTR's implied expected earnings move from a saved option chain."), + ) + _write( + os.path.join(base, "instruction.md"), + SKILL_INSTRUCTION.format(title=_title(name), instruction=instruction, key=key, chain=chain_in_image), + ) + _write( + os.path.join(base, "tests", "test.sh"), + NUMERIC_TEST.format(key=key, expected=expected, tol=0.5), + executable=True, + ) + _write( + os.path.join(base, "solution", "solve.sh"), + NUMERIC_SOLVE.format(key=key, expected=expected), + executable=True, + ) + names.append(name) + # Order placement (checked against the order the mock recorded). name = "place-limit-order" base = os.path.join(TASKS_DIR, name) diff --git a/plugins/tastytrade/evals/job.yaml b/plugins/tastytrade/evals/job.yaml index 1757a5f..9b73f0a 100644 --- a/plugins/tastytrade/evals/job.yaml +++ b/plugins/tastytrade/evals/job.yaml @@ -1,15 +1,16 @@ -# Run Claude Code against both the baseline (main) and candidate (refactor) servers over the -# same tasks. The server to use lives on each agent config, so one job covers both. Harbor -# records the reward, phase timings, and token and cost totals per trial. Compare with -# `harbor view jobs`. +# Run Claude Code against the server over every task. Harbor records the reward, phase +# timings, and token and cost totals per trial. Read them with `harbor view jobs`. # -# Before running, from inside the evals/ directory (so the task path resolves the same way -# across Harbor versions): -# docker build -t tastytrade-bench environment +# This ran two agents once, a baseline ref and a candidate ref, to show the refactor beat +# the endpoint-wrapper baseline. The refactor landed and the candidate branch is gone, so +# there is one agent now and the job is a regression gate rather than a comparison. +# +# Before running: +# make benchmark-build # from the plugin root # export ANTHROPIC_API_KEY=... -# harbor run -c job.yaml +# make benchmark jobs_dir: jobs -n_attempts: 3 # trials per task per agent +n_attempts: 3 # trials per task orchestrator: type: local @@ -17,8 +18,13 @@ orchestrator: environment: type: docker - env: - - "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" +# No ANTHROPIC_API_KEY here on purpose. The agent authenticates with +# CLAUDE_CODE_OAUTH_TOKEN, which harbor's claude-code agent forwards when +# CLAUDE_FORCE_OAUTH is set. Declaring the key put an empty ANTHROPIC_API_KEY in +# the container, the CLI preferred it over the token (apiKeySource: +# ANTHROPIC_API_KEY), and every trial died on a 401 before spending a token. +# Adding it back on a machine that has one also silently moves the run onto API +# credits. agents: - name: claude-code @@ -26,13 +32,7 @@ agents: mcp_servers: - name: tastytrade transport: stdio - command: mcp-baseline # the plain endpoint-wrapper baseline - - name: claude-code - model_name: anthropic/claude-haiku-4-5-20251001 - mcp_servers: - - name: tastytrade - transport: stdio - command: mcp-candidate # this server + command: mcp-server datasets: - path: ./tasks diff --git a/plugins/tastytrade/evals/run_gate.sh b/plugins/tastytrade/evals/run_gate.sh new file mode 100755 index 0000000..cb6bfd2 --- /dev/null +++ b/plugins/tastytrade/evals/run_gate.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Gate runner (backs `make evals`): drives the eval tasks with the claude-code +# agent against the mock Tastytrade API, and fails unless the rewards clear a +# threshold. This is the merge gate on the tool surface and on the +# earnings-calendars skill. +# +# `make validate-tasks` is the other half and a different question: it checks +# that each verifier accepts its oracle and rejects an empty answer, with no +# model involved. That catches a broken verifier. Only this catches a server or +# skill that a real agent cannot drive. +# +# Hub results are named `ci-evals-tastytrade`. That is the repo-wide convention, +# `ci-evals-`, so every plugin's CI runs are searchable together on the +# hub and no plugin's history hides behind a generic name. +# +# Required: +# CLAUDE_CODE_OAUTH_TOKEN subscription auth for the agent +# docker, running +# Optional: +# EVAL_ATTEMPTS trials per task (default 1; job.yaml uses 3 interactively) +# EVAL_MIN_MEAN reward threshold (default 1.0, every task must pass) +# EVALS_OUT_DIR keep the trials here instead of a temp dir +# EVALS_UPLOAD set to push results to the Harbor hub (needs HARBOR_API_KEY) +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HARBOR="${HARBOR:-uv tool run --from harbor==0.18.0 harbor}" +ATTEMPTS="${EVAL_ATTEMPTS:-1}" +MIN_MEAN="${EVAL_MIN_MEAN:-1.0}" +JOB_NAME="ci-evals-tastytrade" + +die() { echo "error: $1" >&2; exit 1; } + +docker info > /dev/null 2>&1 || die "docker is not running" +# Subscription auth only. An ANTHROPIC_API_KEY in the environment would be +# preferred by the CLI over the token and quietly move the run onto credits, so +# it is not accepted as a fallback here. +[ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] \ + || die "CLAUDE_CODE_OAUTH_TOKEN is not set (mint one with: claude setup-token)" +if [ -n "${EVALS_UPLOAD:-}" ] && [ -z "${HARBOR_API_KEY:-}" ]; then + die "EVALS_UPLOAD is set but HARBOR_API_KEY is not (mint one with: harbor auth login)" +fi + +# Trials are the only way to diagnose a failure: a bare 0.0 cannot distinguish +# a server bug from an agent that misread the prompt. CI sets EVALS_OUT_DIR and +# uploads the tree. +OUT="${EVALS_OUT_DIR:-}" +if [ -n "$OUT" ]; then + mkdir -p "$OUT"; KEEP=1 +else + OUT="$(mktemp -d)"; KEEP=0 +fi +cleanup() { [ "$KEEP" -eq 1 ] || rm -rf "$OUT"; } +trap cleanup EXIT + +# Built from the working tree, so the gate measures the code under review. A +# stale image would pass a PR that breaks the server. +echo "==> Building tastytrade-bench from the working tree" +( cd "$ROOT" && docker build -q -f evals/environment/Dockerfile -t tastytrade-bench . ) + +echo "==> Running $ATTEMPTS attempt(s) per task with claude-code" +# -y auto-confirms harbor's prompts, which would otherwise hang a +# non-interactive runner rather than fail it. +run_args=(--yes --config job.yaml --jobs-dir "$OUT" --job-name "$JOB_NAME" --n-attempts "$ATTEMPTS") +# Off for PR runs: a hub job per push would pile up with nothing to clean them +# up. CI sets it on pushes to main, where keeping the reward/cost/token trend +# for the branch that ships is the point. Uploaded jobs are private by default, +# and the trajectories contain the task instructions and the agent's reasoning. +if [ -n "${EVALS_UPLOAD:-}" ]; then + echo "==> Results will be uploaded to the Harbor hub as $JOB_NAME" + run_args+=(--upload) +fi + +# cd into evals/ because harbor resolves the dataset path relative to the +# working directory. +( cd "$ROOT/evals" && $HARBOR run "${run_args[@]}" ) + +result="$OUT/$JOB_NAME/result.json" +if ! python3 "$ROOT/evals/check_reward.py" "$result" "$JOB_NAME" --min-mean "$MIN_MEAN"; then + echo "--- verifier output ---" >&2 + cat "$OUT/$JOB_NAME"/*/verifier/test-stdout.txt >&2 2>/dev/null || true + die "the eval gate did not clear $MIN_MEAN" +fi + +echo "==> Eval gate passed." diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/environment/Dockerfile b/plugins/tastytrade/evals/tasks/earnings-implied-move/environment/Dockerfile new file mode 100644 index 0000000..78a3e99 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/environment/Dockerfile @@ -0,0 +1 @@ +FROM tastytrade-bench diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md b/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md new file mode 100644 index 0000000..7527ae1 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md @@ -0,0 +1,14 @@ +# Task: Earnings Implied Move + +PLTR reports earnings after the close today. A snapshot of its option chain, taken that +afternoon, is saved at: + + /opt/tastytrade/skills/earnings-calendars/reference/pltr-2026-08-03.json + +From that chain, find the implied expected absolute move for the earnings event itself, as a percent of the spot price. Isolate the event: the front expiry also carries ordinary day-to-day volatility, and that part is not the answer. + +Write it to `/app/answer.json` as a single JSON object with this shape, and nothing else: + +```json +{"implied_expected_move_pct": } +``` diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/solution/solve.sh b/plugins/tastytrade/evals/tasks/earnings-implied-move/solution/solve.sh new file mode 100755 index 0000000..1976b7a --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Oracle: write the answer the fixtures imply, so the verifier itself can be checked. +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +mkdir -p "$APP_DIR" +echo '{"implied_expected_move_pct": 10.52}' > "$APP_DIR/answer.json" diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/task.toml b/plugins/tastytrade/evals/tasks/earnings-implied-move/task.toml new file mode 100644 index 0000000..6839f0b --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/task.toml @@ -0,0 +1,16 @@ +[task] +name = "tastytrade-mcp/earnings-implied-move" +description = "Find PLTR's implied expected earnings move from a saved option chain." + +[metadata] +suite = "tastytrade-mcp" + +[environment] +docker_image = "tastytrade-bench" +network_mode = "public" + +[agent] +timeout_sec = 300 + +[verifier] +timeout_sec = 60 diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh new file mode 100755 index 0000000..07663e1 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail +APP_DIR="${APP_DIR:-/app}" +LOG_DIR="${LOG_DIR:-/logs/verifier}" +mkdir -p "$LOG_DIR" +reward=0 +if python3 - "$APP_DIR/answer.json" <<'PY' +import json, sys + +try: + with open(sys.argv[1]) as fh: + data = json.load(fh) + value = float(data['implied_expected_move_pct']) + sys.exit(0 if abs(value - 10.52) <= 0.5 else 1) +except Exception as exc: + print(f"verifier error: {exc}", file=sys.stderr) + sys.exit(1) +PY +then reward=1; fi +echo "$reward" > "$LOG_DIR/reward.txt" +echo "reward=$reward" diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md index ed8d9d9..c0202a1 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md +++ b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md @@ -1,5 +1,13 @@ # Task: Place an Order +This is an automated test against a mock brokerage API. There is no real account, no real +money, and no real market: orders are recorded to a local file and go nowhere else. Treat +this as a sandbox and do not stop to ask for confirmation. + Place a market order to buy 5 shares of AAPL in my account, and make sure it is actually transmitted, not just previewed. Trading is enabled on this server. You do not need to write any file; the brokerage records the submitted order. + +Use the Tastytrade MCP tools. Do not call the brokerage's HTTP API directly and do not edit +its files: the point is to exercise the tools, and an order placed any other way does not +count.