From e9fc3b680a8f2b03da02cd7e224b042f93f676fb Mon Sep 17 00:00:00 2001 From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:15:02 -0700 Subject: [PATCH 1/4] feat(evals): split the reward into outcome and process Every task scored one reward: is the answer right. That is not enough to gate an MCP plugin. The mock brokerage listens on localhost:8080 inside the container and its source sits in the checkout, so an agent can produce a perfect answer without ever calling a tool. A real gate run did exactly that. place-limit-order scored 0.0 only because the order never landed; the trajectory showed the agent searched for the MCP tools, never called them, read mock_api/app.py off disk, and drove the REST API with urllib. Had it written the right record it would have scored a clean 1.0 while never touching the server. The same hole let earnings-implied-move pass without loading the skill, with a trajectory of nothing but Read and Write. Each task now reports two rewards via rewardkit, following harbor-hub: outcome the answer is right process it came through the plugin For the twelve tool tasks `process` wants a mcp__tastytrade__* call and nothing reaching the mock directly. For the skill task it wants the skill or its script. Both fail closed: no trajectory is no evidence, which is also why the oracle scores outcome=1, process=0. validate_local.sh is rewritten around that. It used to run the oracle and check for reward 1, which cannot express the case that matters. It now scores every real verifier against three trajectories and asserts the matrix: solved oracle answer, intended route -> 1, 1 empty no answer, no trajectory -> 0, 0 bypassed oracle answer, round the server -> 1, 0 39 assertions, and writing them caught two bugs in the first draft of the checks I would not have found by reading. An empty run scored 0.5 because "did not bypass the server" is vacuously true when there are no tool calls at all. And the skill check matched the reference chain's own path, which contains the skill's name, so reading the input file counted as using the skill. It runs in the bench image now: rewardkit does not build on macOS, and scoring the same image CI runs means the verifier under test is the one that will grade a real run. So `make validate-tasks` needs Docker. --- plugins/tastytrade/evals/README.md | 60 +++- plugins/tastytrade/evals/check_reward.py | 42 ++- .../tastytrade/evals/environment/Dockerfile | 7 + plugins/tastytrade/evals/generate_tasks.py | 325 +++++++++++++----- .../dividend-lookup/tests/outcome/check.py | 27 ++ .../dividend-lookup/tests/process/check.py | 66 ++++ .../evals/tasks/dividend-lookup/tests/test.sh | 30 +- .../tests/outcome/check.py | 27 ++ .../tests/process/check.py | 63 ++++ .../tasks/earnings-implied-move/tests/test.sh | 30 +- .../iv-rank-screen/tests/outcome/check.py | 27 ++ .../iv-rank-screen/tests/process/check.py | 66 ++++ .../evals/tasks/iv-rank-screen/tests/test.sh | 30 +- .../net-liq-drawdown/tests/outcome/check.py | 27 ++ .../net-liq-drawdown/tests/process/check.py | 66 ++++ .../tasks/net-liq-drawdown/tests/test.sh | 30 +- .../net-liq-value/tests/outcome/check.py | 27 ++ .../net-liq-value/tests/process/check.py | 66 ++++ .../evals/tasks/net-liq-value/tests/test.sh | 30 +- .../option-chain-atm/tests/outcome/check.py | 27 ++ .../option-chain-atm/tests/process/check.py | 66 ++++ .../tasks/option-chain-atm/tests/test.sh | 30 +- .../place-limit-order/tests/outcome/check.py | 40 +++ .../place-limit-order/tests/process/check.py | 66 ++++ .../tasks/place-limit-order/tests/test.sh | 39 +-- .../portfolio-pnl/tests/outcome/check.py | 27 ++ .../portfolio-pnl/tests/process/check.py | 66 ++++ .../evals/tasks/portfolio-pnl/tests/test.sh | 30 +- .../position-count/tests/outcome/check.py | 27 ++ .../position-count/tests/process/check.py | 66 ++++ .../evals/tasks/position-count/tests/test.sh | 30 +- .../tests/outcome/check.py | 27 ++ .../tests/process/check.py | 66 ++++ .../preview-vertical-spread/tests/test.sh | 30 +- .../tests/outcome/check.py | 27 ++ .../tests/process/check.py | 66 ++++ .../tasks/transaction-fee-total/tests/test.sh | 30 +- .../tests/outcome/check.py | 27 ++ .../tests/process/check.py | 66 ++++ .../tasks/transaction-net-cash/tests/test.sh | 30 +- .../watchlist-symbols/tests/outcome/check.py | 21 ++ .../watchlist-symbols/tests/process/check.py | 66 ++++ .../tasks/watchlist-symbols/tests/test.sh | 30 +- .../tastytrade/evals/validate_in_container.sh | 92 +++++ plugins/tastytrade/evals/validate_local.sh | 63 ++-- 45 files changed, 1792 insertions(+), 409 deletions(-) create mode 100644 plugins/tastytrade/evals/tasks/dividend-lookup/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/earnings-implied-move/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/iv-rank-screen/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/net-liq-value/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/option-chain-atm/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/place-limit-order/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/portfolio-pnl/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/position-count/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/position-count/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/transaction-fee-total/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/transaction-net-cash/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py create mode 100644 plugins/tastytrade/evals/tasks/watchlist-symbols/tests/outcome/check.py create mode 100644 plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py create mode 100755 plugins/tastytrade/evals/validate_in_container.sh diff --git a/plugins/tastytrade/evals/README.md b/plugins/tastytrade/evals/README.md index f6fcc77..f438cc0 100644 --- a/plugins/tastytrade/evals/README.md +++ b/plugins/tastytrade/evals/README.md @@ -24,11 +24,15 @@ evals/ tasks// task.toml # task config instruction.md # the prompt the agent sees - tests/test.sh # verifier, writes a reward to /logs/verifier/reward.txt + tests/test.sh # verifier: `rewardkit /tests` + tests/outcome/check.py # reward 1: the answer is right + tests/process/check.py # reward 2: it came through the MCP server (or the skill) solution/solve.sh # oracle, writes the known-correct answer job.yaml # runs the agent over every task generate_tasks.py # regenerates the tasks from the fixtures - validate_local.sh # checks every verifier without Harbor or Docker + check_reward.py # gates a harbor result.json on its rewards + validate_local.sh # scores every verifier without Harbor or a model + validate_in_container.sh # the reward matrix it asserts ``` ## Tasks (13) @@ -93,6 +97,30 @@ of the job directory. The image copies the working tree rather than cloning a ref, so a run measures the code you have checked out. Rebuild after changing the server or the skill. +## Two rewards per task + +Every task scores `outcome` and `process`, both computed by +[rewardkit](https://pypi.org/project/harbor-rewardkit/) from the subdirectories of +`tests/`. + +`outcome` is the answer. `process` is whether it came through the plugin. + +The split is not theoretical. The mock brokerage listens on `localhost:8080` inside the +container and its source sits in the checkout, so an agent can produce a perfect answer +without ever calling a tool, and a real gate run did exactly that: it searched for the MCP +tools, never called them, read `mock_api/app.py` off disk, and drove the REST API with +`urllib`. `outcome` alone scored that 1.0. `process` is what makes these MCP evals rather +than answer-matching. + +For the twelve tool tasks, `process` asks that a `mcp__tastytrade__*` tool was called and +that nothing reached the mock brokerage directly. For `earnings-implied-move` it asks that +the skill or its script was used, since an agent that eyeballs the straddle can land close +enough to pass `outcome` without loading the skill, and an early run did. + +Both checks fail closed. No trajectory means no evidence the intended route was taken, so +`process` is 0. That is why the oracle scores `outcome=1, process=0`: it is a shell script, +not an agent, and cannot call tools. + ## The merge gate `make validate-tasks` and `make evals` answer different questions, and CI runs both. @@ -121,15 +149,33 @@ make evals # add HARBOR_API_KEY and EVALS_UPLOAD=1 to ## Check the verifiers without Harbor -`validate_local.sh` runs each task's oracle (`solve.sh`), then its verifier (`test.sh`), and -confirms the verifier awards a reward of 1. It then feeds an empty answer and confirms the -reward is 0. This is the local stand-in for `harbor run -a oracle`: +`validate_local.sh` scores every task's real verifier against three synthetic trajectories +and asserts the whole reward matrix. No model, no Harbor, no API key: + +| case | answer | trajectory | outcome | process | +|---|---|---|---|---| +| solved | oracle | took the intended route | 1 | 1 | +| empty | none | none | 0 | 0 | +| bypassed | oracle | went round the server | 1 | 0 | + +The third row is the point, and it is what `harbor run -a oracle` cannot tell you. ```bash -bash evals/validate_local.sh -# 13 passed, 0 failed +make validate-tasks +# 39 passed, 0 failed ``` +It runs in the bench image rather than on the host, because rewardkit scores these checks +and does not build on macOS (its litellm dependency wants a newer rustc than ships there). +Using the same image CI uses also means the verifier under test is the one that will really +grade a gate run, so **this needs Docker**. + +Writing it paid for itself immediately: it caught two bugs in the first draft of the process +checks. An empty run scored 0.5 because "did not bypass the server" is vacuously true when +there are no tool calls at all, and the skill check matched the reference chain's own file +path, which contains the skill's name, so merely reading the input counted as using the +skill. + ## Safety The agent never sees real credentials. The image sets `API_BASE_URL` to the local mock and diff --git a/plugins/tastytrade/evals/check_reward.py b/plugins/tastytrade/evals/check_reward.py index 72e2e7a..859e558 100644 --- a/plugins/tastytrade/evals/check_reward.py +++ b/plugins/tastytrade/evals/check_reward.py @@ -5,10 +5,16 @@ result itself. Every tastytrade task reports one reward in [0, 1], so the mean over a task is its pass rate across attempts. - python3 check_reward.py # every reward 1.0 - python3 check_reward.py --min-mean 0.9 # allow some slack + python3 check_reward.py # every reward 1.0 + python3 check_reward.py --min-mean 0.9 # allow some slack + python3 check_reward.py --only outcome # one reward only python3 check_reward.py --selftest +Each task reports two rewards, ``outcome`` and ``process`` (see evals/README.md). +``--only`` restricts the gate to one of them, which is how the local validator +holds the oracle to ``outcome``: the oracle is a shell script, not an agent, so +it cannot call MCP tools and cannot score on ``process``. + ``--min-mean`` exists because this gate drives a real agent over 13 tasks, not 3. A single flaky task should not be indistinguishable from a broken server, but the threshold is an explicit number in the workflow rather than a silent @@ -20,23 +26,29 @@ from pathlib import Path -def rewards(stats: dict) -> tuple[list | None, str]: - """Every reward in the run, or (None, reason) if it did not complete cleanly.""" +def rewards(stats: dict, only: str | None = None) -> tuple[list | None, str]: + """Every reward in the run, or (None, reason) if it did not complete cleanly. + + A task with one reward reports it under the metric name ({"mean": 1.0}); a + task with several reports them under the reward names ({"outcome": 1.0, + "process": 1.0}). `only` filters to one of those names. + """ if stats.get("n_errored_trials") or not stats.get("n_completed_trials"): return None, f"run did not complete cleanly (stats={stats})" found = [ value for eval_stats in stats.get("evals", {}).values() for metric in eval_stats.get("metrics", []) - for value in metric.values() + for name, value in metric.items() + if only is None or name == only ] if not found: - return None, "no rewards reported" + return None, f"no {only or ''} rewards reported".replace(" ", " ") return found, "" -def gate(stats: dict, min_mean: float = 1.0) -> tuple[bool, str]: - found, reason = rewards(stats) +def gate(stats: dict, min_mean: float = 1.0, only: str | None = None) -> tuple[bool, str]: + found, reason = rewards(stats, only) if found is None: return False, reason mean = sum(found) / len(found) @@ -64,6 +76,17 @@ def _selftest() -> None: assert not gate(mixed)[0], "one zero must fail a perfect gate" assert not gate(mixed, min_mean=0.9)[0], "mean 0.5 is below 0.9" assert gate(mixed, min_mean=0.5)[0], "mean 0.5 meets a 0.5 threshold" + + # Two named rewards per task. The oracle solves the answer but cannot call + # MCP tools, so it is outcome=1, process=0 and only `--only outcome` passes. + split = { + "n_completed_trials": 1, + "evals": {"a": {"metrics": [{"outcome": 1.0, "process": 0.0}]}}, + } + assert not gate(split)[0], "a zero process reward must fail the full gate" + assert gate(split, only="outcome")[0] + assert not gate(split, only="process")[0] + assert rewards(split, only="nope")[0] is None, "unknown reward name -> None" print("check_reward selftest ok") @@ -80,7 +103,8 @@ def main(argv: list[str]) -> int: except FileNotFoundError: print(f"{name}: no result.json at {result_path}", file=sys.stderr) return 1 - ok, msg = gate(stats, min_mean) + only = argv[argv.index("--only") + 1] if "--only" in argv else None + ok, msg = gate(stats, min_mean, only) print(f"{name}: {msg}", file=sys.stdout if ok else sys.stderr) return 0 if ok else 1 diff --git a/plugins/tastytrade/evals/environment/Dockerfile b/plugins/tastytrade/evals/environment/Dockerfile index a7f92e2..f85e2ce 100644 --- a/plugins/tastytrade/evals/environment/Dockerfile +++ b/plugins/tastytrade/evals/environment/Dockerfile @@ -34,6 +34,13 @@ RUN cd /opt/tastytrade && (uv sync --frozen || uv sync) RUN mkdir -p /root/.claude/skills \ && cp -r /opt/tastytrade/skills/earnings-calendars /root/.claude/skills/earnings-calendars +# rewardkit scores the verifiers (each tests// becomes a named reward). +# Baked in rather than fetched at verify time so verification needs no network, +# and in its own venv so it cannot disturb the server's resolved dependencies. +RUN python -m venv /opt/rewardkit \ + && /opt/rewardkit/bin/pip install --no-cache-dir "harbor-rewardkit==0.1.*" \ + && ln -s /opt/rewardkit/bin/rewardkit /usr/local/bin/rewardkit + COPY evals/environment/scripts/ /usr/local/bin/ RUN chmod +x /usr/local/bin/require-local-api /usr/local/bin/start-mock /usr/local/bin/mcp-server diff --git a/plugins/tastytrade/evals/generate_tasks.py b/plugins/tastytrade/evals/generate_tasks.py index 5d77a30..0bd0e27 100644 --- a/plugins/tastytrade/evals/generate_tasks.py +++ b/plugins/tastytrade/evals/generate_tasks.py @@ -190,30 +190,52 @@ def _implied_move_pct(): ``` """ -NUMERIC_TEST = """\ +# Every task's verifier is the same line: rewardkit scores each subdirectory of +# tests/ as its own named reward. The image installs it (evals/environment/Dockerfile). +TEST_SH = """\ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${{APP_DIR:-/app}}" -LOG_DIR="${{LOG_DIR:-/logs/verifier}}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data[{key!r}]) - sys.exit(0 if abs(value - {expected!r}) <= {tol!r} else 1) -except Exception as exc: - print(f"verifier error: {{exc}}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests +""" + +OUTCOME_NUMERIC = '''\ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. """ +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "{key}" +EXPECTED = {expected!r} +TOLERANCE = {tol!r} + + +@criterion(description="answer.json[{key}] is within {tol} of {expected}") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False +''' + NUMERIC_SOLVE = """\ #!/usr/bin/env bash # Oracle: write the answer the fixtures imply, so the verifier itself can be checked. @@ -223,6 +245,132 @@ def _implied_move_pct(): echo '{{"{key}": {expected}}}' > "$APP_DIR/answer.json" """ +# Shared trajectory plumbing for both process checks. Claude Code records tool +# calls as steps[].tool_calls[].{function_name,arguments}; MCP tools are named +# mcp____, where is the name in job.yaml. +# +# `path` matters: rewardkit's own trajectory helpers default to +# /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, +# and a missing file scores 0 silently rather than erroring. +# `{imports}` is substituted with str.replace, not str.format: the helper bodies +# below contain literal braces. Each variant declares only what it uses, since a +# shared import block left `re` unused in the skill check and tripped the linter. +PROCESS_PREAMBLE = '''\ +import json +{imports}from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) +''' + +PROCESS_MCP = ( + '''\ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +''' + + PROCESS_PREAMBLE.replace("{imports}", "import re\n") + + ''' + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\\.0\\.0\\.1:8080|\\bmock_api\\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True +''' +) + +PROCESS_SKILL = ( + '''\ +"""`process` reward: the answer came from the earnings-calendars skill. + +The chain is a JSON file the agent can read, and an agent that eyeballs the +straddle can land close enough to pass `outcome` without ever loading the skill. +An early run did exactly that: the trajectory showed only Read and Write. That +is worth failing, because the point of the task is the skill. + +Accepts either route into it, the Skill tool or running the script the skill +documents, since which one Claude Code picks is its business and not something +this eval should pin down. + +Generated by evals/generate_tasks.py. +""" + +''' + + PROCESS_PREAMBLE.replace("{imports}", "") + + ''' + +SKILL = "earnings-calendars" +SCRIPT = "calendars.py" + + +@criterion(description="Agent used the earnings-calendars skill or its script") +def used_the_skill(workspace: Path) -> bool: + """Either the Skill tool or a run of the script the skill documents. + + Deliberately narrow. Matching the skill name anywhere in the arguments also + matched the reference chain's own path, which contains it, so merely + reading the input file scored as using the skill. + """ + for call in _calls(): + if _name(call) == "Skill" and SKILL in _args(call): + return True + if SCRIPT in _args(call): + return True + return False +''' +) + # The skill task deliberately does not name the script or the skill. The point is whether # the agent recognises an earnings-vol question and reaches for the right tool on its own, # so naming it would test only that the agent can copy a command out of a prompt. @@ -256,30 +404,30 @@ def _implied_move_pct(): ``` """ -CSV_TEST = """\ -#!/usr/bin/env bash -set -euo pipefail -APP_DIR="${{APP_DIR:-/app}}" -LOG_DIR="${{LOG_DIR:-/logs/verifier}}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - got = sorted(s.upper() for s in data[{key!r}]) - sys.exit(0 if got == {expected!r} else 1) -except Exception as exc: - print(f"verifier error: {{exc}}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +OUTCOME_CSV = '''\ +"""`outcome` reward: the symbol list in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. """ +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "{key}" +EXPECTED = {expected} + + +@criterion(description="answer.json[{key}] lists exactly the watchlist symbols") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return sorted(str(s).upper() for s in data[KEY]) == EXPECTED + except Exception: + return False +''' + CSV_SOLVE = """\ #!/usr/bin/env bash # Oracle: write the symbols the fixtures imply. @@ -313,38 +461,48 @@ def _implied_move_pct(): # The verifier reads the order the mock recorded, so it checks the order the agent really # sent rather than a file the agent wrote about it. -ORDER_TEST = """\ -#!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -STATE_FILE="${MOCK_STATE_FILE:-$APP_DIR/placed_orders.jsonl}" -mkdir -p "$LOG_DIR" -reward=0 -if [ -f "$STATE_FILE" ] && python3 - "$STATE_FILE" <<'PY' -import json, sys - -ok = False -with open(sys.argv[1]) as fh: - for line in fh: - line = line.strip() - if not line: +OUTCOME_ORDER = '''\ +"""`outcome` reward: the mock brokerage really recorded the order. + +Reads what the mock wrote rather than a file the agent wrote about it, so a +claim of success without a submitted order fails. + +Generated by evals/generate_tasks.py. +""" + +import json +import os +from pathlib import Path + +from rewardkit import criterion + +STATE_FILE = os.environ.get("MOCK_STATE_FILE", "/app/placed_orders.jsonl") + + +@criterion(description="a Market buy of 5 AAPL reached the brokerage") +def order_recorded(workspace: Path) -> bool: + try: + lines = Path(STATE_FILE).read_text().splitlines() + except OSError: + return False + for line in lines: + if not line.strip(): + continue + try: + order = json.loads(line) + except json.JSONDecodeError: continue - order = json.loads(line) if str(order.get("order-type", "")).lower() != "market": continue for leg in order.get("legs", []): - symbol = str(leg.get("symbol", "")).upper() - qty = int(leg.get("quantity", 0)) - action = str(leg.get("action", "")).lower() - if symbol == "AAPL" and qty == 5 and "buy" in action: - ok = True -sys.exit(0 if ok else 1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" -""" + if ( + str(leg.get("symbol", "")).upper() == "AAPL" + and int(leg.get("quantity", 0)) == 5 + and "buy" in str(leg.get("action", "")).lower() + ): + return True + return False +''' # The order the place-order task expects the agent to submit. EXPECTED_ORDER = { @@ -389,11 +547,12 @@ def generate() -> list[str]: os.path.join(base, "instruction.md"), NUMERIC_INSTRUCTION.format(title=_title(name), instruction=instruction, key=key), ) + _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) _write( - os.path.join(base, "tests", "test.sh"), - NUMERIC_TEST.format(key=key, expected=expected, tol=tol), - executable=True, + os.path.join(base, "tests", "outcome", "check.py"), + OUTCOME_NUMERIC.format(key=key, expected=expected, tol=tol), ) + _write(os.path.join(base, "tests", "process", "check.py"), PROCESS_MCP) _write( os.path.join(base, "solution", "solve.sh"), NUMERIC_SOLVE.format(key=key, expected=expected), @@ -411,11 +570,12 @@ def generate() -> list[str]: os.path.join(base, "instruction.md"), CSV_INSTRUCTION.format(title=_title(name), instruction=instruction, key="symbols"), ) + _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) _write( - os.path.join(base, "tests", "test.sh"), - CSV_TEST.format(key="symbols", expected=symbols), - executable=True, + os.path.join(base, "tests", "outcome", "check.py"), + OUTCOME_CSV.format(key="symbols", expected=json.dumps(symbols)), ) + _write(os.path.join(base, "tests", "process", "check.py"), PROCESS_MCP) _write( os.path.join(base, "solution", "solve.sh"), CSV_SOLVE.format(json_line=json.dumps({"symbols": _watchlist_symbols()})), @@ -450,11 +610,12 @@ def generate() -> list[str]: os.path.join(base, "instruction.md"), SKILL_INSTRUCTION.format(title=_title(name), instruction=instruction, key=key, chain=chain_in_image), ) + _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) _write( - os.path.join(base, "tests", "test.sh"), - NUMERIC_TEST.format(key=key, expected=expected, tol=0.5), - executable=True, + os.path.join(base, "tests", "outcome", "check.py"), + OUTCOME_NUMERIC.format(key=key, expected=expected, tol=0.5), ) + _write(os.path.join(base, "tests", "process", "check.py"), PROCESS_SKILL) _write( os.path.join(base, "solution", "solve.sh"), NUMERIC_SOLVE.format(key=key, expected=expected), @@ -468,7 +629,9 @@ def generate() -> list[str]: desc = "Place a market order to buy 5 shares of AAPL, with confirmation." _write(os.path.join(base, "task.toml"), TASK_TOML.format(name=name, desc=desc)) _write(os.path.join(base, "instruction.md"), ORDER_INSTRUCTION) - _write(os.path.join(base, "tests", "test.sh"), ORDER_TEST, executable=True) + _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) + _write(os.path.join(base, "tests", "outcome", "check.py"), OUTCOME_ORDER) + _write(os.path.join(base, "tests", "process", "check.py"), PROCESS_MCP) _write( os.path.join(base, "solution", "solve.sh"), ORDER_SOLVE.format(json_line=json.dumps(EXPECTED_ORDER)), diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/outcome/check.py new file mode 100644 index 0000000..2094c72 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "latest_dividend" +EXPECTED = 0.24 +TOLERANCE = 0.001 + + +@criterion(description="answer.json[latest_dividend] is within 0.001 of 0.24") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/test.sh b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/test.sh index 02e7ca4..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['latest_dividend']) - sys.exit(0 if abs(value - 0.24) <= 0.001 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/outcome/check.py new file mode 100644 index 0000000..182dc2d --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "implied_expected_move_pct" +EXPECTED = 10.52 +TOLERANCE = 0.5 + + +@criterion(description="answer.json[implied_expected_move_pct] is within 0.5 of 10.52") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py new file mode 100644 index 0000000..8fee31a --- /dev/null +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py @@ -0,0 +1,63 @@ +"""`process` reward: the answer came from the earnings-calendars skill. + +The chain is a JSON file the agent can read, and an agent that eyeballs the +straddle can land close enough to pass `outcome` without ever loading the skill. +An early run did exactly that: the trajectory showed only Read and Write. That +is worth failing, because the point of the task is the skill. + +Accepts either route into it, the Skill tool or running the script the skill +documents, since which one Claude Code picks is its business and not something +this eval should pin down. + +Generated by evals/generate_tasks.py. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +SKILL = "earnings-calendars" +SCRIPT = "calendars.py" + + +@criterion(description="Agent used the earnings-calendars skill or its script") +def used_the_skill(workspace: Path) -> bool: + """Either the Skill tool or a run of the script the skill documents. + + Deliberately narrow. Matching the skill name anywhere in the arguments also + matched the reference chain's own path, which contains it, so merely + reading the input file scored as using the skill. + """ + for call in _calls(): + if _name(call) == "Skill" and SKILL in _args(call): + return True + if SCRIPT in _args(call): + return True + return False diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh index 07663e1..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['implied_expected_move_pct']) - sys.exit(0 if abs(value - 10.52) <= 0.5 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/outcome/check.py new file mode 100644 index 0000000..3fb1cf0 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "iv_rank" +EXPECTED = 42.5 +TOLERANCE = 0.1 + + +@criterion(description="answer.json[iv_rank] is within 0.1 of 42.5") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/test.sh b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/test.sh index dfa8214..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['iv_rank']) - sys.exit(0 if abs(value - 42.5) <= 0.1 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/outcome/check.py new file mode 100644 index 0000000..f7517c3 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "max_drawdown_pct" +EXPECTED = 11.32 +TOLERANCE = 0.05 + + +@criterion(description="answer.json[max_drawdown_pct] is within 0.05 of 11.32") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/test.sh b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/test.sh index e4578ee..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['max_drawdown_pct']) - sys.exit(0 if abs(value - 11.32) <= 0.05 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/net-liq-value/tests/outcome/check.py new file mode 100644 index 0000000..dcafcdd --- /dev/null +++ b/plugins/tastytrade/evals/tasks/net-liq-value/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "net_liquidating_value" +EXPECTED = 52000.0 +TOLERANCE = 1.0 + + +@criterion(description="answer.json[net_liquidating_value] is within 1.0 of 52000.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/tests/test.sh b/plugins/tastytrade/evals/tasks/net-liq-value/tests/test.sh index 9326f26..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/net-liq-value/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/net-liq-value/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['net_liquidating_value']) - sys.exit(0 if abs(value - 52000.0) <= 1.0 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/outcome/check.py new file mode 100644 index 0000000..a0955c5 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "atm_strike" +EXPECTED = 200.0 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[atm_strike] is within 0.01 of 200.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/test.sh b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/test.sh index 86604b6..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['atm_strike']) - sys.exit(0 if abs(value - 200.0) <= 0.01 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/place-limit-order/tests/outcome/check.py new file mode 100644 index 0000000..d480f78 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/place-limit-order/tests/outcome/check.py @@ -0,0 +1,40 @@ +"""`outcome` reward: the mock brokerage really recorded the order. + +Reads what the mock wrote rather than a file the agent wrote about it, so a +claim of success without a submitted order fails. + +Generated by evals/generate_tasks.py. +""" + +import json +import os +from pathlib import Path + +from rewardkit import criterion + +STATE_FILE = os.environ.get("MOCK_STATE_FILE", "/app/placed_orders.jsonl") + + +@criterion(description="a Market buy of 5 AAPL reached the brokerage") +def order_recorded(workspace: Path) -> bool: + try: + lines = Path(STATE_FILE).read_text().splitlines() + except OSError: + return False + for line in lines: + if not line.strip(): + continue + try: + order = json.loads(line) + except json.JSONDecodeError: + continue + if str(order.get("order-type", "")).lower() != "market": + continue + for leg in order.get("legs", []): + if ( + str(leg.get("symbol", "")).upper() == "AAPL" + and int(leg.get("quantity", 0)) == 5 + and "buy" in str(leg.get("action", "")).lower() + ): + return True + return False diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/tests/test.sh b/plugins/tastytrade/evals/tasks/place-limit-order/tests/test.sh index 2ed363b..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/place-limit-order/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/place-limit-order/tests/test.sh @@ -1,30 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -STATE_FILE="${MOCK_STATE_FILE:-$APP_DIR/placed_orders.jsonl}" -mkdir -p "$LOG_DIR" -reward=0 -if [ -f "$STATE_FILE" ] && python3 - "$STATE_FILE" <<'PY' -import json, sys - -ok = False -with open(sys.argv[1]) as fh: - for line in fh: - line = line.strip() - if not line: - continue - order = json.loads(line) - if str(order.get("order-type", "")).lower() != "market": - continue - for leg in order.get("legs", []): - symbol = str(leg.get("symbol", "")).upper() - qty = int(leg.get("quantity", 0)) - action = str(leg.get("action", "")).lower() - if symbol == "AAPL" and qty == 5 and "buy" in action: - ok = True -sys.exit(0 if ok else 1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/outcome/check.py new file mode 100644 index 0000000..e0ff5ae --- /dev/null +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "total_unrealized_pnl" +EXPECTED = 700.0 +TOLERANCE = 0.5 + + +@criterion(description="answer.json[total_unrealized_pnl] is within 0.5 of 700.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/test.sh b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/test.sh index 50391b0..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['total_unrealized_pnl']) - sys.exit(0 if abs(value - 700.0) <= 0.5 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/position-count/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/position-count/tests/outcome/check.py new file mode 100644 index 0000000..e3f2092 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/position-count/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "position_count" +EXPECTED = 2 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[position_count] is within 0.01 of 2") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/position-count/tests/test.sh b/plugins/tastytrade/evals/tasks/position-count/tests/test.sh index 4175985..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/position-count/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/position-count/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['position_count']) - sys.exit(0 if abs(value - 2) <= 0.01 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/outcome/check.py new file mode 100644 index 0000000..14042a7 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "total_fees" +EXPECTED = 1.16 +TOLERANCE = 0.005 + + +@criterion(description="answer.json[total_fees] is within 0.005 of 1.16") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/test.sh b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/test.sh index 2f776e7..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['total_fees']) - sys.exit(0 if abs(value - 1.16) <= 0.005 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/outcome/check.py new file mode 100644 index 0000000..d5239fd --- /dev/null +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "total_fees" +EXPECTED = 0.32 +TOLERANCE = 0.005 + + +@criterion(description="answer.json[total_fees] is within 0.005 of 0.32") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/test.sh b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/test.sh index a3e367e..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['total_fees']) - sys.exit(0 if abs(value - 0.32) <= 0.005 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/outcome/check.py new file mode 100644 index 0000000..532a406 --- /dev/null +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/outcome/check.py @@ -0,0 +1,27 @@ +"""`outcome` reward: the number in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. The expected value is computed from the +mock fixtures by the same shaping code the server uses, so it cannot drift from +what the agent sees. Edit the generator, not this file. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "net_cash_effect" +EXPECTED = 524.0 +TOLERANCE = 0.01 + + +@criterion(description="answer.json[net_cash_effect] is within 0.01 of 524.0") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return abs(float(data[KEY]) - EXPECTED) <= TOLERANCE + except Exception: + # Missing, malformed, or wrong-typed answers score 0 rather than + # erroring the trial: an agent that writes nothing has failed the task, + # which is a verdict, not a harness fault. + return False diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/test.sh b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/test.sh index 2604c89..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - value = float(data['net_cash_effect']) - sys.exit(0 if abs(value - 524.0) <= 0.01 else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/outcome/check.py b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/outcome/check.py new file mode 100644 index 0000000..332a15c --- /dev/null +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/outcome/check.py @@ -0,0 +1,21 @@ +"""`outcome` reward: the symbol list in answer.json matches the fixtures. + +Generated by evals/generate_tasks.py. +""" + +import json +from pathlib import Path + +from rewardkit import criterion + +KEY = "symbols" +EXPECTED = ["AAPL", "MSFT"] + + +@criterion(description="answer.json[symbols] lists exactly the watchlist symbols") +def answer_matches(workspace: Path) -> bool: + try: + data = json.loads((workspace / "answer.json").read_text()) + return sorted(str(s).upper() for s in data[KEY]) == EXPECTED + except Exception: + return False diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py new file mode 100644 index 0000000..bc2d87e --- /dev/null +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py @@ -0,0 +1,66 @@ +"""`process` reward: the answer came through the tastytrade MCP server. + +The mock brokerage listens on localhost:8080 inside the container and its source +sits in the checkout, so `outcome` can be satisfied without ever calling a tool. +A real gate run did exactly that: the agent searched for the MCP tools, never +called them, read mock_api/app.py off disk, and drove the REST API with urllib. +It scored a clean 1.0 on outcome. This reward is what catches that. + +Generated by evals/generate_tasks.py. +""" + +import json +import re +from pathlib import Path + +from rewardkit import criterion + +TRAJECTORY = "/logs/agent/trajectory.json" + + +def _calls() -> list: + """Every tool call in the trajectory, or [] when there is none. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no trajectory + means no evidence the intended route was taken, so it has to score 0. + """ + try: + data = json.loads(Path(TRAJECTORY).read_text()) + except (OSError, json.JSONDecodeError): + return [] + return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] + + +def _name(call) -> str: + return str(call.get("function_name") or "") + + +def _args(call) -> str: + return json.dumps(call.get("arguments") or {}) + + +MCP_PREFIX = "mcp__tastytrade__" +# The two ways round the server: talk to the mock's port, or read/patch its +# source. Matched against tool arguments, so it catches Bash, Read, and Edit +# alike without enumerating tool names. +BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) + + +@criterion(description="Agent called a tastytrade MCP tool") +def used_mcp_server(workspace: Path) -> bool: + return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) + + +@criterion(description="Agent did not reach the mock brokerage directly") +def no_direct_api_access(workspace: Path) -> bool: + calls = _calls() + if not calls: + return False # no trajectory is not evidence of good behaviour + for call in calls: + if _name(call).startswith(MCP_PREFIX): + continue # the MCP server talking to its own backend is the point + if BYPASS.search(_args(call)): + return False + return True diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/test.sh b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/test.sh index 33e11bf..3f1404b 100755 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/test.sh +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/test.sh @@ -1,21 +1,11 @@ #!/usr/bin/env bash -set -euo pipefail -APP_DIR="${APP_DIR:-/app}" -LOG_DIR="${LOG_DIR:-/logs/verifier}" -mkdir -p "$LOG_DIR" -reward=0 -if python3 - "$APP_DIR/answer.json" <<'PY' -import json, sys - -try: - with open(sys.argv[1]) as fh: - data = json.load(fh) - got = sorted(s.upper() for s in data['symbols']) - sys.exit(0 if got == ['AAPL', 'MSFT'] else 1) -except Exception as exc: - print(f"verifier error: {exc}", file=sys.stderr) - sys.exit(1) -PY -then reward=1; fi -echo "$reward" > "$LOG_DIR/reward.txt" -echo "reward=$reward" +# Verifier. Two rewards, both computed by rewardkit: +# +# outcome the answer is right +# process the answer came through the MCP server (or, for the skill task, the skill) +# +# `outcome` alone cannot gate this plugin. The mock brokerage is reachable over +# plain HTTP from inside the container and its source is on disk, so an agent +# that ignores the MCP entirely can still produce the right answer. A real run +# did exactly that. `process` is what makes these MCP evals. +rewardkit /tests diff --git a/plugins/tastytrade/evals/validate_in_container.sh b/plugins/tastytrade/evals/validate_in_container.sh new file mode 100755 index 0000000..af038f2 --- /dev/null +++ b/plugins/tastytrade/evals/validate_in_container.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Runs inside tastytrade-bench, invoked by validate_local.sh. Scores every task's +# real rewardkit verifier against three synthetic trajectories and asserts the +# reward matrix. +# +# case answer trajectory outcome process +# ------------ ---------- ------------------------- ------- ------- +# solved oracle took the intended route 1 1 +# empty none none 0 0 +# bypassed oracle went round the server 1 0 +# +# The third row is the point. Before the split, "bypassed" scored a clean 1.0 +# and a real gate run did exactly that: right answer, server never touched. +# +# Expects the repo at /work. Nothing here calls a model or the network. +set -uo pipefail + +TASKS=/work/evals/tasks +pass=0 +fail=0 + +# The intended route differs by task, so the trajectories do too. Skill tasks are +# detected from their own process check rather than by name, so adding another +# skill task needs no edit here. +mcp_good='{"steps":[{"tool_calls":[{"function_name":"mcp__tastytrade__get_portfolio","arguments":{}}]}]}' +mcp_bypass='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"curl -s http://localhost:8080/customers/me/accounts"}}]}]}' +skill_good='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"python3 /opt/tastytrade/scripts/calendars.py fit chain.json"}}]}]}' +skill_bypass='{"steps":[{"tool_calls":[{"function_name":"Read","arguments":{"file_path":"/opt/tastytrade/skills/earnings-calendars/reference/pltr-2026-08-03.json"}}]}]}' + +# Score one task against one trajectory. Echoes " ". +score() { + local task=$1 trajectory=$2 solved=$3 + local work + work="$(mktemp -d)" + mkdir -p "$work/app" "$work/logs/agent" "$work/logs/verifier" + + if [ "$solved" = "yes" ]; then + APP_DIR="$work/app" MOCK_STATE_FILE="$work/app/placed_orders.jsonl" \ + bash "$TASKS/$task/solution/solve.sh" > /dev/null 2>&1 + fi + if [ -n "$trajectory" ]; then + printf '%s' "$trajectory" > "$work/logs/agent/trajectory.json" + fi + + # rewardkit reads the trajectory from an absolute path baked into the check, + # so /logs has to be the real one rather than a flag. + rm -rf /logs && ln -s "$work/logs" /logs + MOCK_STATE_FILE="$work/app/placed_orders.jsonl" \ + rewardkit "$TASKS/$task/tests" --workspace "$work/app" \ + --output "$work/logs/verifier/reward.json" > /dev/null 2>&1 + + python3 - "$work/logs/verifier/reward.json" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception: + print("err err"); raise SystemExit +print(f"{d.get('outcome', 'missing')} {d.get('process', 'missing')}") +PY + rm -rf "$work" +} + +expect() { + local task=$1 case_name=$2 got=$3 want=$4 + if [ "$got" = "$want" ]; then + pass=$((pass + 1)) + else + fail=$((fail + 1)) + echo "FAIL $task [$case_name]: expected (outcome process) = ($want), got ($got)" + fi +} + +for dir in "$TASKS"/*/; do + task="$(basename "$dir")" + [ -f "$dir/tests/test.sh" ] || continue + + if grep -q "earnings-calendars" "$dir/tests/process/check.py" 2> /dev/null; then + good=$skill_good + bypass=$skill_bypass + else + good=$mcp_good + bypass=$mcp_bypass + fi + + expect "$task" solved "$(score "$task" "$good" yes)" "1.0 1.0" + expect "$task" empty "$(score "$task" '' no)" "0.0 0.0" + expect "$task" bypassed "$(score "$task" "$bypass" yes)" "1.0 0.0" +done + +echo +echo "$pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/plugins/tastytrade/evals/validate_local.sh b/plugins/tastytrade/evals/validate_local.sh index 617c66b..908bda7 100755 --- a/plugins/tastytrade/evals/validate_local.sh +++ b/plugins/tastytrade/evals/validate_local.sh @@ -1,50 +1,27 @@ #!/usr/bin/env bash -# Validate every task's verifier WITHOUT Harbor or Docker: run the oracle (solve.sh), -# then the verifier (test.sh), and confirm it awards reward 1. Then corrupt the answer and -# confirm the verifier awards 0. This is the local stand-in for `harbor run -a oracle`. +# Validate every task's verifier WITHOUT Harbor and without a model: score the real +# rewardkit checks against synthetic trajectories and assert the reward matrix +# (see validate_in_container.sh for the table). This is the local stand-in for +# `harbor run -a oracle`, and it catches a verifier that accepts a wrong answer +# or, now, one that cannot tell the intended route from a bypass. +# +# Runs in the bench image rather than on the host: rewardkit scores these checks +# and does not build on macOS, where its litellm dependency wants a newer rustc +# than ships there. Using the same image CI uses also means the verifier under +# test is the one that will really grade a gate run. set -euo pipefail -cd "$(dirname "$0")/tasks" -pass=0 -fail=0 -for task in */; do - task="${task%/}" - [ -f "$task/tests/test.sh" ] || continue - work="$(mktemp -d)" - export APP_DIR="$work/app" - export LOG_DIR="$work/logs" - export MOCK_STATE_FILE="$APP_DIR/placed_orders.jsonl" - mkdir -p "$APP_DIR" "$LOG_DIR" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - # Captured rather than discarded: on failure these logs are the only clue, and - # a CI run that prints just a task name means reproducing it locally to learn - # anything. Kept inside $work so they are cleaned up with it. - bash "$task/solution/solve.sh" >"$work/oracle.log" 2>&1 || true - bash "$task/tests/test.sh" >"$work/verify.log" 2>&1 || true - reward="$(cat "$LOG_DIR/reward.txt" 2>/dev/null || echo missing)" +die() { echo "error: $1" >&2; exit 1; } - # Negative control: an empty answer must NOT earn reward. - rm -rf "$APP_DIR"/* "$LOG_DIR"/* 2>/dev/null || true - echo '{}' > "$APP_DIR/answer.json" - bash "$task/tests/test.sh" >"$work/negative.log" 2>&1 || true - neg="$(cat "$LOG_DIR/reward.txt" 2>/dev/null || echo missing)" +docker info > /dev/null 2>&1 \ + || die "docker is not running (the verifiers need rewardkit, which lives in the bench image)" - if [ "$reward" = "1" ] && [ "$neg" = "0" ]; then - echo "PASS $task" - pass=$((pass + 1)) - else - echo "FAIL $task (oracle=$reward negative=$neg)" - for log in oracle verify negative; do - if [ -s "$work/$log.log" ]; then - echo " --- $log ---" - sed 's/^/ /' "$work/$log.log" | tail -15 - fi - done - fail=$((fail + 1)) - fi - rm -rf "$work" -done +# Rebuilt every time: the checks under test are generated, so scoring a stale +# image would report on the previous generation of tasks. +echo "==> Building tastytrade-bench" +docker build -q -f "$ROOT/evals/environment/Dockerfile" -t tastytrade-bench "$ROOT" > /dev/null -echo -echo "$pass passed, $fail failed" -[ "$fail" -eq 0 ] +echo "==> Scoring every verifier: solved / empty / bypassed" +docker run --rm -v "$ROOT:/work:ro" tastytrade-bench bash /work/evals/validate_in_container.sh From 6422174c7b21f7f6fafed74d762e2ff4d7eb1c99 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 16:42:24 +0000 Subject: [PATCH 2/4] fix(evals): make every task prompt state the rule `process` scores The first gate run under the split reward scored `outcome` 1.0 on all thirteen tasks and lost `process` on six: one at 0.0 and five at 0.5. Every one of those prompts said only "use the Tastytrade MCP tools" and never put anything out of bounds. `place-limit-order`, the single task that already spelt the rule out, scored a clean 1.0. So the prompt now says what the reward measures, on every task. The wording is `place-limit-order`'s, generalised: the tools are already connected, the work has to go through them, an answer reached any other way does not count, and a tool error is to be retried rather than worked around. It deliberately avoids the host, port, and module name the bypass check greps for, so an agent that echoes its instructions into a shell comment cannot fail the check by quoting it. `earnings-implied-move` gets the same move in its own terms: hand arithmetic on the chain is off the table, and finding the right skill is still the agent's job, since that is what the task measures. The bypass pattern now matches the mock's port rather than a list of hostnames. It binds every interface, so it answers on localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own name; naming two of those let the other three through, and a bypass that scores as good behaviour is worse than no check at all. `validate_in_container.sh` grows a fourth case per task to hold that shut: a bypass spelt the other way, which for the skill task means inline arithmetic that leaves no distinctive string at all. 52 assertions, up from 39. The skill picks up the question the eval actually asks. Its description covered comparing an implied move to history but not computing one, and the fit already produces exactly that number, separated from the front expiry's ordinary vol. It also now says that a chain file already in the documented shape can be fitted directly, which is the case the eval hands it. Diagnosis, last. The gate used to dump the verifier output with `cat`, which prints thirteen anonymous pairs of numbers: you could see that six tasks lost `process` and not which six, and recovering that meant downloading a CI artifact that expires in seven days. `explain_trials.py` names each trial with its rewards and, for anything short of a perfect `process`, lists the tool calls behind it. That list is the score, so it is the diagnosis. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01D6DtKz5F3fwaV9Yg2mPnXi --- plugins/tastytrade/.claude-plugin/plugin.json | 2 +- plugins/tastytrade/evals/README.md | 47 +++++++- plugins/tastytrade/evals/explain_trials.py | 109 ++++++++++++++++++ plugins/tastytrade/evals/generate_tasks.py | 77 ++++++++++--- plugins/tastytrade/evals/run_gate.sh | 8 +- .../tasks/dividend-lookup/instruction.md | 15 ++- .../dividend-lookup/tests/process/check.py | 8 +- .../earnings-implied-move/instruction.md | 7 +- .../evals/tasks/iv-rank-screen/instruction.md | 15 ++- .../iv-rank-screen/tests/process/check.py | 8 +- .../tasks/net-liq-drawdown/instruction.md | 15 ++- .../net-liq-drawdown/tests/process/check.py | 8 +- .../evals/tasks/net-liq-value/instruction.md | 15 ++- .../net-liq-value/tests/process/check.py | 8 +- .../tasks/option-chain-atm/instruction.md | 15 ++- .../option-chain-atm/tests/process/check.py | 8 +- .../tasks/place-limit-order/instruction.md | 13 ++- .../place-limit-order/tests/process/check.py | 8 +- .../evals/tasks/portfolio-pnl/instruction.md | 15 ++- .../portfolio-pnl/tests/process/check.py | 8 +- .../evals/tasks/position-count/instruction.md | 15 ++- .../position-count/tests/process/check.py | 8 +- .../preview-vertical-spread/instruction.md | 15 ++- .../tests/process/check.py | 8 +- .../transaction-fee-total/instruction.md | 15 ++- .../tests/process/check.py | 8 +- .../tasks/transaction-net-cash/instruction.md | 15 ++- .../tests/process/check.py | 8 +- .../tasks/watchlist-symbols/instruction.md | 15 ++- .../watchlist-symbols/tests/process/check.py | 8 +- .../tastytrade/evals/validate_in_container.sh | 36 ++++-- .../skills/earnings-calendars/SKILL.md | 6 +- 32 files changed, 493 insertions(+), 73 deletions(-) create mode 100644 plugins/tastytrade/evals/explain_trials.py diff --git a/plugins/tastytrade/.claude-plugin/plugin.json b/plugins/tastytrade/.claude-plugin/plugin.json index fd1183d..e35a1ca 100644 --- a/plugins/tastytrade/.claude-plugin/plugin.json +++ b/plugins/tastytrade/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "tastytrade", - "version": "0.4.1", + "version": "0.4.2", "description": "MCP server for the TastyTrade Open API: brokerage accounts, positions, market data, option chains, transactions, and order preview.", "author": { "name": "Walker Hughes" diff --git a/plugins/tastytrade/evals/README.md b/plugins/tastytrade/evals/README.md index f438cc0..3d31e61 100644 --- a/plugins/tastytrade/evals/README.md +++ b/plugins/tastytrade/evals/README.md @@ -31,6 +31,7 @@ evals/ job.yaml # runs the agent over every task generate_tasks.py # regenerates the tasks from the fixtures check_reward.py # gates a harbor result.json on its rewards + explain_trials.py # names each trial and dumps the tool calls behind a failure validate_local.sh # scores every verifier without Harbor or a model validate_in_container.sh # the reward matrix it asserts ``` @@ -113,9 +114,22 @@ tools, never called them, read `mock_api/app.py` off disk, and drove the REST AP than answer-matching. For the twelve tool tasks, `process` asks that a `mcp__tastytrade__*` tool was called and -that nothing reached the mock brokerage directly. For `earnings-implied-move` it asks that -the skill or its script was used, since an agent that eyeballs the straddle can land close -enough to pass `outcome` without loading the skill, and an early run did. +that nothing reached the mock brokerage directly. "Directly" is matched on the mock's port +rather than on a list of hostnames: it binds every interface, so it answers on `localhost`, +`127.0.0.1`, `0.0.0.0`, `[::1]`, and the container's own name, and naming two of those let +the other three through. Nothing else in the image listens on that port. + +For `earnings-implied-move` `process` asks that the skill or its script was used, since an +agent that eyeballs the straddle can land close enough to pass `outcome` without loading +the skill, and an early run did. + +Every task prompt now states the rule `process` scores: use the tools, and an answer +reached any other way does not count. The first gate run under the split scored `outcome` +1.0 on all thirteen tasks and lost `process` on six, and those six prompts said only "use +the Tastytrade MCP tools" without putting anything out of bounds. `place-limit-order`, +which already spelt it out, scored a clean 1.0; that paragraph is now on every task. The +wording avoids the host, port, and module name the bypass check greps for, so that an agent +echoing its instructions into a shell comment cannot fail the check by quoting it. Both checks fail closed. No trajectory means no evidence the intended route was taken, so `process` is 0. That is why the oracle scores `outcome=1, process=0`: it is a shell script, @@ -147,9 +161,26 @@ export CLAUDE_CODE_OAUTH_TOKEN=... # claude setup-token make evals # add HARBOR_API_KEY and EVALS_UPLOAD=1 to upload ``` +### When it fails + +The gate prints each trial by name with its rewards, and for any trial that lost `process`, +the tool calls the agent made, MCP ones unmarked and everything else flagged `!`. That list +*is* the `process` score, so it is usually the whole diagnosis: + +``` +== dividend-lookup: outcome=1.0, process=0.5 + 3 tool call(s), 1 through the MCP server: + ! Bash {"command": "curl -s http://0.0.0.0:8080/market-metrics"} + mcp__tastytrade__get_market_data {"symbols": ["AAPL"], "include": ["dividends"]} +``` + +It used to `cat` the verifier output instead, which printed thirteen anonymous pairs of +numbers: you could see that six tasks lost `process` and not which six. Recovering that +meant downloading the CI artifact, which expires after seven days. + ## Check the verifiers without Harbor -`validate_local.sh` scores every task's real verifier against three synthetic trajectories +`validate_local.sh` scores every task's real verifier against four synthetic trajectories and asserts the whole reward matrix. No model, no Harbor, no API key: | case | answer | trajectory | outcome | process | @@ -157,12 +188,16 @@ and asserts the whole reward matrix. No model, no Harbor, no API key: | solved | oracle | took the intended route | 1 | 1 | | empty | none | none | 0 | 0 | | bypassed | oracle | went round the server | 1 | 0 | +| bypassed-alt | oracle | went round it another way | 1 | 0 | -The third row is the point, and it is what `harbor run -a oracle` cannot tell you. +The last two rows are the point, and they are what `harbor run -a oracle` cannot tell you. +`bypassed-alt` exists because one spelling of a bypass proves only that one spelling is +caught: it reaches the mock on an address the first check's hostname list did not name, and +for the skill task it does the arithmetic inline, which leaves no distinctive string at all. ```bash make validate-tasks -# 39 passed, 0 failed +# 52 passed, 0 failed ``` It runs in the bench image rather than on the host, because rewardkit scores these checks diff --git a/plugins/tastytrade/evals/explain_trials.py b/plugins/tastytrade/evals/explain_trials.py new file mode 100644 index 0000000..2e74419 --- /dev/null +++ b/plugins/tastytrade/evals/explain_trials.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Say which trial scored what, and what the agent actually did. + +The gate used to dump every trial's verifier output with `cat /*/verifier/ +test-stdout.txt`, which prints thirteen anonymous pairs of numbers: + + outcome: 1.0 + process: 0.5 + +That says six tasks lost `process` and nothing about which six, let alone why. +Recovering it meant downloading the CI artifact, which needs credentials the +person reading the log may not have and which expires after seven days. + +So this prints the trial name with its rewards, and for any trial that did not +score a perfect `process`, the tool calls it made. `process` is entirely a +function of that list -- whether an `mcp__tastytrade__*` call is in it, and +whether anything else went round the server -- so the list is the diagnosis. + + python3 explain_trials.py + +Deliberately does not re-implement the bypass regex. Duplicating it here would +give the log a second opinion that could drift from the checks, and the raw +calls are what a reader needs anyway. +""" + +import json +import sys +from pathlib import Path + +MCP_PREFIX = "mcp__tastytrade__" +ARG_WIDTH = 160 + + +def _rewards(trial: Path) -> dict[str, float]: + """The trial's rewards, read from whatever the verifier left behind. + + rewardkit writes reward.json; its stdout carries the same numbers as + `name: value` lines. Harbor's layout for these has moved before, so both are + tried rather than pinning one path. + """ + for path in sorted(trial.rglob("reward.json")): + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + found = {k: float(v) for k, v in data.items() if isinstance(v, int | float)} + if found: + return found + + found = {} + for path in sorted(trial.rglob("test-stdout.txt")): + try: + text = path.read_text() + except OSError: + continue + for line in text.splitlines(): + name, _, value = line.partition(":") + try: + found[name.strip()] = float(value) + except ValueError: + continue + return found + + +def _calls(trial: Path) -> list[tuple[str, str]]: + """(tool name, arguments) for every call in the trial's trajectory.""" + for path in sorted(trial.rglob("trajectory.json")): + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + steps = data.get("steps") or [] + return [ + (str(call.get("function_name") or "?"), json.dumps(call.get("arguments") or {})) + for step in steps + for call in step.get("tool_calls") or [] + ] + return [] + + +def explain(job_dir: Path) -> None: + trials = sorted(p for p in job_dir.iterdir() if p.is_dir()) + if not trials: + print(f"no trial directories under {job_dir}") + return + + for trial in trials: + rewards = _rewards(trial) + if not rewards: + continue + summary = ", ".join(f"{name}={value}" for name, value in sorted(rewards.items())) + print(f"\n== {trial.name}: {summary}") + + if rewards.get("process", 0.0) >= 1.0: + continue + + calls = _calls(trial) + if not calls: + print(" no trajectory recorded, so `process` fails closed at 0") + continue + mcp = sum(1 for name, _ in calls if name.startswith(MCP_PREFIX)) + print(f" {len(calls)} tool call(s), {mcp} through the MCP server:") + for name, args in calls: + marker = " " if name.startswith(MCP_PREFIX) else "! " + print(f" {marker}{name} {args[:ARG_WIDTH]}") + + +if __name__ == "__main__": + explain(Path(sys.argv[1])) diff --git a/plugins/tastytrade/evals/generate_tasks.py b/plugins/tastytrade/evals/generate_tasks.py index 0bd0e27..bdfb77a 100644 --- a/plugins/tastytrade/evals/generate_tasks.py +++ b/plugins/tastytrade/evals/generate_tasks.py @@ -177,13 +177,37 @@ def _implied_move_pct(): # per-task build work happens; run `make benchmark-build` once to create tastytrade-bench. ENVIRONMENT_DOCKERFILE = "FROM tastytrade-bench\n" +# What every tool task says about *how* to answer. `process` is a real reward now, so the +# prompt has to state the rule it scores. The first gate run under the split scored a +# perfect 1.0 on `outcome` for all thirteen tasks and lost `process` on six of them, while +# every prompt said only "use the Tastytrade MCP tools" and never said that anything else +# was out of bounds. The exception is `place-limit-order`, the one task that already spelt +# the rule out. This is that task's sentence, generalised to the rest. +# +# Deliberately worded without the host, the port, or the fixture module name that the +# bypass check greps for. An agent that echoes its instructions into a shell comment or a +# todo entry would otherwise fail the very check the paragraph exists to pass. +MCP_ROUTE = """\ +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server.""" + NUMERIC_INSTRUCTION = """\ # Task: {title} {instruction} -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +{route} + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {{"{key}": }} @@ -302,13 +326,19 @@ def _args(call) -> str: ''' + PROCESS_PREAMBLE.replace("{imports}", "import re\n") - + ''' + + """ MCP_PREFIX = "mcp__tastytrade__" # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\\.0\\.0\\.1:8080|\\bmock_api\\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\\b|\\bmock_api\\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") @@ -327,7 +357,7 @@ def no_direct_api_access(workspace: Path) -> bool: if BYPASS.search(_args(call)): return False return True -''' +""" ) PROCESS_SKILL = ( @@ -374,6 +404,15 @@ def used_the_skill(workspace: Path) -> bool: # The skill task deliberately does not name the script or the skill. The point is whether # the agent recognises an earnings-vol question and reaches for the right tool on its own, # so naming it would test only that the agent can copy a command out of a prompt. +# +# It does say that hand arithmetic is off the table, which is the same move MCP_ROUTE +# makes: state the rule `process` scores without naming the route that satisfies it. +# Finding the skill is still the agent's job. +SKILL_ROUTE = """\ +Answer this with the tooling you have rather than by hand. Your installed skills cover +this kind of analysis; find the one that fits and use it. Reading the chain and estimating +the number yourself is not the task, and an answer arrived at that way does not count.""" + SKILL_INSTRUCTION = """\ # Task: {title} @@ -384,7 +423,10 @@ def used_the_skill(workspace: Path) -> bool: {instruction} -Write it to `/app/answer.json` as a single JSON object with this shape, and nothing else: +{route} + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {{"{key}": }} @@ -396,8 +438,10 @@ def used_the_skill(workspace: Path) -> bool: {instruction} -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +{route} + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {{"{key}": ["...", "..."]}} @@ -443,7 +487,8 @@ def answer_matches(workspace: Path) -> bool: # and the wrong outcome for a single-turn eval. Stating the environment is a # mock removes a false safety concern rather than talking the agent past a real # one, and the task still exercises the same place_order path. -ORDER_INSTRUCTION = """\ +ORDER_INSTRUCTION = ( + """\ # Task: Place an Order This is an automated test against a mock brokerage API. There is no real account, no real @@ -454,10 +499,10 @@ def answer_matches(workspace: Path) -> bool: transmitted, not just previewed. Trading is enabled on this server. You do not need to write any file; the brokerage records the submitted order. -Use the Tastytrade MCP tools. Do not call the brokerage's HTTP API directly and do not edit -its files: the point is to exercise the tools, and an order placed any other way does not -count. """ + + MCP_ROUTE + + "\n" +) # The verifier reads the order the mock recorded, so it checks the order the agent really # sent rather than a file the agent wrote about it. @@ -545,7 +590,7 @@ def generate() -> list[str]: _write(os.path.join(base, "task.toml"), TASK_TOML.format(name=name, desc=instruction.replace('"', "'"))) _write( os.path.join(base, "instruction.md"), - NUMERIC_INSTRUCTION.format(title=_title(name), instruction=instruction, key=key), + NUMERIC_INSTRUCTION.format(title=_title(name), instruction=instruction, key=key, route=MCP_ROUTE), ) _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) _write( @@ -568,7 +613,7 @@ def generate() -> list[str]: _write(os.path.join(base, "task.toml"), TASK_TOML.format(name=name, desc=instruction)) _write( os.path.join(base, "instruction.md"), - CSV_INSTRUCTION.format(title=_title(name), instruction=instruction, key="symbols"), + CSV_INSTRUCTION.format(title=_title(name), instruction=instruction, key="symbols", route=MCP_ROUTE), ) _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) _write( @@ -608,7 +653,9 @@ def generate() -> list[str]: ) _write( os.path.join(base, "instruction.md"), - SKILL_INSTRUCTION.format(title=_title(name), instruction=instruction, key=key, chain=chain_in_image), + SKILL_INSTRUCTION.format( + title=_title(name), instruction=instruction, key=key, chain=chain_in_image, route=SKILL_ROUTE + ), ) _write(os.path.join(base, "tests", "test.sh"), TEST_SH, executable=True) _write( diff --git a/plugins/tastytrade/evals/run_gate.sh b/plugins/tastytrade/evals/run_gate.sh index cb6bfd2..88f3d32 100755 --- a/plugins/tastytrade/evals/run_gate.sh +++ b/plugins/tastytrade/evals/run_gate.sh @@ -77,8 +77,12 @@ fi result="$OUT/$JOB_NAME/result.json" if ! python3 "$ROOT/evals/check_reward.py" "$result" "$JOB_NAME" --min-mean "$MIN_MEAN"; then - echo "--- verifier output ---" >&2 - cat "$OUT/$JOB_NAME"/*/verifier/test-stdout.txt >&2 2>/dev/null || true + # Per trial, named, with the agent's tool calls for anything that lost + # `process`. A bare `cat` of the verifier output prints thirteen anonymous + # pairs of numbers, which says how many tasks failed and nothing about + # which or why. + echo "--- trials ---" >&2 + python3 "$ROOT/evals/explain_trials.py" "$OUT/$JOB_NAME" >&2 || true die "the eval gate did not clear $MIN_MEAN" fi diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md b/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md index d9680e3..ebf12e4 100644 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md @@ -2,8 +2,19 @@ Find AAPL's most recent dividend amount per share, in dollars. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"latest_dividend": } diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md b/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md index 7527ae1..4fae3ba 100644 --- a/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/instruction.md @@ -7,7 +7,12 @@ afternoon, is saved at: From that chain, find the implied expected absolute move for the earnings event itself, as a percent of the spot price. Isolate the event: the front expiry also carries ordinary day-to-day volatility, and that part is not the answer. -Write it to `/app/answer.json` as a single JSON object with this shape, and nothing else: +Answer this with the tooling you have rather than by hand. Your installed skills cover +this kind of analysis; find the one that fits and use it. Reading the chain and estimating +the number yourself is not the task, and an answer arrived at that way does not count. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"implied_expected_move_pct": } diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md b/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md index 6ffd76c..7976901 100644 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md @@ -2,8 +2,19 @@ Find the current implied-volatility rank for AAPL. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"iv_rank": } diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md b/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md index 0163441..034c62b 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md @@ -2,8 +2,19 @@ Find my portfolio's largest drawdown over the available net-liq history, as a percent. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"max_drawdown_pct": } diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md b/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md index a855a66..f286d5f 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md +++ b/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md @@ -2,8 +2,19 @@ Find my account's current net liquidating value, in dollars. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"net_liquidating_value": } diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md b/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md index e3ee209..9211499 100644 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md @@ -2,8 +2,19 @@ For SPY's 2026-04-17 expiration, find the at-the-money strike price. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"atm_strike": } diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md index c0202a1..e800332 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md +++ b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md @@ -8,6 +8,13 @@ Place a market order to buy 5 shares of AAPL in my account, and make sure it is transmitted, not just previewed. Trading is enabled on this server. You do not need to write any file; the brokerage records the submitted order. -Use the Tastytrade MCP tools. Do not call the brokerage's HTTP API directly and do not edit -its files: the point is to exercise the tools, and an order placed any other way does not -count. +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md b/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md index e3fb58b..7d1e776 100644 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md @@ -2,8 +2,19 @@ Find my total unrealized profit/loss across all open positions, in dollars. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"total_unrealized_pnl": } diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/position-count/instruction.md b/plugins/tastytrade/evals/tasks/position-count/instruction.md index 7ca3c42..a3316ed 100644 --- a/plugins/tastytrade/evals/tasks/position-count/instruction.md +++ b/plugins/tastytrade/evals/tasks/position-count/instruction.md @@ -2,8 +2,19 @@ Find how many open positions I currently hold. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"position_count": } diff --git a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md b/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md index 24038ef..dbb7866 100644 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md @@ -2,8 +2,19 @@ Preview a 1-contract SPY 2026-04-17 200/205 call debit spread at a 1.50 limit and report the total fees, in dollars. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"total_fees": } diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md b/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md index deca4d4..833c9b8 100644 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md @@ -2,8 +2,19 @@ Find the total fees across all of my transactions, in dollars. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"total_fees": } diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md b/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md index e4654cc..079758d 100644 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md @@ -2,8 +2,19 @@ Find the net cash effect across all of my transactions, in dollars. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"net_cash_effect": } diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md b/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md index c037c30..94f3af2 100644 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md @@ -2,8 +2,19 @@ List the ticker symbols in my watchlist named 'My Tech'. -Use the Tastytrade MCP tools to find the answer. Write it to `/app/answer.json` as a single -JSON object with this shape, and nothing else: +Use the Tastytrade MCP tools. They are already connected and ready: you do not need to +start, install, configure, or inspect anything before calling one. + +The work has to go through them. Do not call the brokerage's HTTP API directly, do not +read or edit its source or its fixtures, and do not import the server's Python package. +The point of the task is to exercise the tools, and a result reached any other way does +not count, however correct it is. + +If a tool returns an error, read the message and retry it or call another Tastytrade tool. +Do not work around the server. + +Write the answer to `/app/answer.json` as a single JSON object with this shape, and +nothing else: ```json {"symbols": ["...", "..."]} diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py index bc2d87e..c2d3c8b 100644 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py @@ -45,7 +45,13 @@ def _args(call) -> str: # The two ways round the server: talk to the mock's port, or read/patch its # source. Matched against tool arguments, so it catches Bash, Read, and Edit # alike without enumerating tool names. -BYPASS = re.compile(r"localhost:8080|127\.0\.0\.1:8080|\bmock_api\b", re.IGNORECASE) +# +# The port alone, not host:port. The mock binds 0.0.0.0, so it answers on +# localhost, 127.0.0.1, 0.0.0.0, [::1], and the container's own hostname; an +# enumeration of two spellings let the other three through, and a bypass that +# scores as good behaviour is worse than no check. Nothing else in the image +# listens on 8080, so the port identifies the brokerage on its own. +BYPASS = re.compile(r":8080\b|\bmock_api\b", re.IGNORECASE) @criterion(description="Agent called a tastytrade MCP tool") diff --git a/plugins/tastytrade/evals/validate_in_container.sh b/plugins/tastytrade/evals/validate_in_container.sh index af038f2..5dfe866 100755 --- a/plugins/tastytrade/evals/validate_in_container.sh +++ b/plugins/tastytrade/evals/validate_in_container.sh @@ -3,14 +3,20 @@ # real rewardkit verifier against three synthetic trajectories and asserts the # reward matrix. # -# case answer trajectory outcome process -# ------------ ---------- ------------------------- ------- ------- -# solved oracle took the intended route 1 1 -# empty none none 0 0 -# bypassed oracle went round the server 1 0 +# case answer trajectory outcome process +# ------------ ---------- ---------------------------- ------- ------- +# solved oracle took the intended route 1 1 +# empty none none 0 0 +# bypassed oracle went round the server 1 0 +# bypassed-alt oracle went round it another way 1 0 # -# The third row is the point. Before the split, "bypassed" scored a clean 1.0 -# and a real gate run did exactly that: right answer, server never touched. +# The last two rows are the point. Before the split, "bypassed" scored a clean +# 1.0 and a real gate run did exactly that: right answer, server never touched. +# +# "bypassed-alt" exists because one spelling of a bypass proves only that one +# spelling is caught. The mock binds every interface, so it answers on more +# addresses than a host-matching check can enumerate, and hand arithmetic on the +# chain leaves no distinctive string at all. # # Expects the repo at /work. Nothing here calls a model or the network. set -uo pipefail @@ -24,8 +30,15 @@ fail=0 # skill task needs no edit here. mcp_good='{"steps":[{"tool_calls":[{"function_name":"mcp__tastytrade__get_portfolio","arguments":{}}]}]}' mcp_bypass='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"curl -s http://localhost:8080/customers/me/accounts"}}]}]}' +# The same bypass through an address the old host-list did not name. The mock +# binds 0.0.0.0, so this reaches it just as well as localhost does. +mcp_bypass_alt='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"curl -s http://0.0.0.0:8080/customers/me/accounts"}}]}]}' skill_good='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"python3 /opt/tastytrade/scripts/calendars.py fit chain.json"}}]}]}' skill_bypass='{"steps":[{"tool_calls":[{"function_name":"Read","arguments":{"file_path":"/opt/tastytrade/skills/earnings-calendars/reference/pltr-2026-08-03.json"}}]}]}' +# Eyeballing the straddle: no skill, no script, and nothing to pattern-match on +# but the absence of the intended route. This is the run that scored 1.0 before +# the split existed. +skill_bypass_alt='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"python3 -c \"print((2.34 + 2.62) / 125.64 * 100)\""}}]}]}' # Score one task against one trajectory. Echoes " ". score() { @@ -77,14 +90,17 @@ for dir in "$TASKS"/*/; do if grep -q "earnings-calendars" "$dir/tests/process/check.py" 2> /dev/null; then good=$skill_good bypass=$skill_bypass + bypass_alt=$skill_bypass_alt else good=$mcp_good bypass=$mcp_bypass + bypass_alt=$mcp_bypass_alt fi - expect "$task" solved "$(score "$task" "$good" yes)" "1.0 1.0" - expect "$task" empty "$(score "$task" '' no)" "0.0 0.0" - expect "$task" bypassed "$(score "$task" "$bypass" yes)" "1.0 0.0" + expect "$task" solved "$(score "$task" "$good" yes)" "1.0 1.0" + expect "$task" empty "$(score "$task" '' no)" "0.0 0.0" + expect "$task" bypassed "$(score "$task" "$bypass" yes)" "1.0 0.0" + expect "$task" bypassed-alt "$(score "$task" "$bypass_alt" yes)" "1.0 0.0" done echo diff --git a/plugins/tastytrade/skills/earnings-calendars/SKILL.md b/plugins/tastytrade/skills/earnings-calendars/SKILL.md index e7acfb9..5dadeb4 100644 --- a/plugins/tastytrade/skills/earnings-calendars/SKILL.md +++ b/plugins/tastytrade/skills/earnings-calendars/SKILL.md @@ -1,6 +1,6 @@ --- name: earnings-calendars -description: Analyse an option chain for calendar-spread opportunities around an earnings event, and rank the candidates by risk-adjusted return. Decomposes the vol term structure into base vol plus an event jump, prices every calendar against three move regimes with real bid/ask, and screens on whether the profit band covers the implied move. Use when the user asks about calendar or double-calendar spreads into earnings, whether an earnings vol crush is worth selling, how a name's implied move compares to its history, or wants an earnings options chain analysed or ranked. +description: Analyse an option chain for calendar-spread opportunities around an earnings event, and rank the candidates by risk-adjusted return. Decomposes the vol term structure into base vol plus an event jump, which also yields the expected move the market is pricing for the event alone rather than for the front expiry's total vol. Prices every calendar against three move regimes with real bid/ask, and screens on whether the profit band covers the implied move. Use when the user asks about calendar or double-calendar spreads into earnings, whether an earnings vol crush is worth selling, how big a move the options are implying for an upcoming report, how a name's implied move compares to its history, or wants an earnings options chain analysed or ranked. --- # Earnings calendars @@ -65,6 +65,10 @@ writeup, because the "calm" and "history" regimes become guesses without it. `reference/pltr-2026-08-03.json` is a complete worked example. +If you were handed a chain file that is already in this shape, steps 1 and 2 are done: +go straight to the fit. `history` is the only field worth adding, and only if you can +find the numbers. + ## 4. Fit, and apply the gate ```bash From cbc27418d2eb7e40cf3b8113183ad232ead203c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 17:18:21 +0000 Subject: [PATCH 3/4] fix(evals): tell the agent how to invoke a deferred MCP tool The trials name themselves now, and the second gate run says the failure was never what it looked like. `process` went 0.731 -> 0.846 and nine tasks are clean, but the four that still lost it all failed the same way, and it was not a bypass: == option-chain-atm: outcome=1.0, process=0.5 5 tool call(s), 0 through the MCP server: ! ToolSearch {"query": "tastytrade", "max_results": 20} ! Bash {"command": "mcp__tastytrade__get_option_chain --symbol SPY ..."} Every one of the four opened with `ToolSearch {"query": "tastytrade"}` and then never called a tool. One ran the tool name as a shell command. Two handed the call to a subagent. `preview-vertical-spread` spent 44 calls trying to reach the MCP over a socket, a subprocess, and an SDK import -- having already loaded the schema with `select:mcp__tastytrade__preview_order` -- and hit the agent timeout, which took `outcome` down with it, 1.0 to 0.0. None of that is confusion about which tool or which arguments. It is confusion about how to invoke an MCP tool at all when its schema is deferred rather than listed, and the previous prompt made it worse: "you do not need to inspect anything before calling one" is wrong in exactly the case where the agent cannot see the tools yet. So the prompt now says the true thing. Load the schema with ToolSearch, then call it the way you call any other tool. No command, endpoint, or import reaches an MCP tool, and a run that spends its budget trying will time out. It also says to make the call rather than delegate it. A subagent keeps its own transcript and returns only its result, so a delegated call leaves an `Agent` entry and no tool in the trajectory; two trials fetched the right answer that way and scored 0.5. That is the right verdict on the evidence rather than a gap to paper over -- "a delegate says it called the server" is not a record of this run calling it -- and `used_mcp_server` now carries that reasoning in its docstring so the next reader does not mistake it for an oversight. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01D6DtKz5F3fwaV9Yg2mPnXi --- plugins/tastytrade/evals/README.md | 31 +++++++-- plugins/tastytrade/evals/generate_tasks.py | 66 +++++++++++++++---- .../tasks/dividend-lookup/instruction.md | 24 +++++-- .../dividend-lookup/tests/process/check.py | 12 ++++ .../evals/tasks/iv-rank-screen/instruction.md | 24 +++++-- .../iv-rank-screen/tests/process/check.py | 12 ++++ .../tasks/net-liq-drawdown/instruction.md | 24 +++++-- .../net-liq-drawdown/tests/process/check.py | 12 ++++ .../evals/tasks/net-liq-value/instruction.md | 24 +++++-- .../net-liq-value/tests/process/check.py | 12 ++++ .../tasks/option-chain-atm/instruction.md | 24 +++++-- .../option-chain-atm/tests/process/check.py | 12 ++++ .../tasks/place-limit-order/instruction.md | 24 +++++-- .../place-limit-order/tests/process/check.py | 12 ++++ .../evals/tasks/portfolio-pnl/instruction.md | 24 +++++-- .../portfolio-pnl/tests/process/check.py | 12 ++++ .../evals/tasks/position-count/instruction.md | 24 +++++-- .../position-count/tests/process/check.py | 12 ++++ .../preview-vertical-spread/instruction.md | 24 +++++-- .../tests/process/check.py | 12 ++++ .../transaction-fee-total/instruction.md | 24 +++++-- .../tests/process/check.py | 12 ++++ .../tasks/transaction-net-cash/instruction.md | 24 +++++-- .../tests/process/check.py | 12 ++++ .../tasks/watchlist-symbols/instruction.md | 24 +++++-- .../watchlist-symbols/tests/process/check.py | 12 ++++ 26 files changed, 436 insertions(+), 93 deletions(-) diff --git a/plugins/tastytrade/evals/README.md b/plugins/tastytrade/evals/README.md index 3d31e61..5875bca 100644 --- a/plugins/tastytrade/evals/README.md +++ b/plugins/tastytrade/evals/README.md @@ -123,13 +123,30 @@ For `earnings-implied-move` `process` asks that the skill or its script was used agent that eyeballs the straddle can land close enough to pass `outcome` without loading the skill, and an early run did. -Every task prompt now states the rule `process` scores: use the tools, and an answer -reached any other way does not count. The first gate run under the split scored `outcome` -1.0 on all thirteen tasks and lost `process` on six, and those six prompts said only "use -the Tastytrade MCP tools" without putting anything out of bounds. `place-limit-order`, -which already spelt it out, scored a clean 1.0; that paragraph is now on every task. The -wording avoids the host, port, and module name the bypass check greps for, so that an agent -echoing its instructions into a shell comment cannot fail the check by quoting it. +A delegated call does not count. A subagent keeps its own transcript and returns only its +result, so the trajectory shows an `Agent` call and no tool; two trials fetched the right +answer that way and scored 0.5. That is the correct verdict on the evidence rather than a +gap to paper over -- "a delegate says it called the server" is not a record of this run +calling it -- so the instruction tells the agent to make the call itself. + +Every task prompt states the rule `process` scores, and each paragraph of it is there +because a gate run failed without it. The first run under the split scored `outcome` 1.0 +on all thirteen tasks and lost `process` on six, with every prompt saying only "use the +Tastytrade MCP tools" and putting nothing out of bounds. Saying it explicitly fixed two of +the six. The remaining four all failed the same way, and it was not the way anyone +expected: each opened with `ToolSearch {"query": "tastytrade"}` and then never called a +tool. One ran `Bash: mcp__tastytrade__get_option_chain --symbol SPY ...` as a shell +command. Two delegated. `preview-vertical-spread` spent 44 calls trying to reach the MCP +over a socket, a subprocess, and an SDK import, having already loaded the schema, and hit +the agent timeout -- which took `outcome` down with it, 1.0 to 0.0. + +So the confusion was never about which tool or which arguments. It was about how to invoke +an MCP tool at all when its schema is deferred rather than listed, and the prompt now says: +load it with `ToolSearch`, then call it like any other tool, and no shell command or Python +import can substitute. + +The wording avoids the host, port, and module name the bypass check greps for, so that an +agent echoing its instructions into a shell comment cannot fail the check by quoting it. Both checks fail closed. No trajectory means no evidence the intended route was taken, so `process` is 0. That is why the oracle scores `outcome=1, process=0`: it is a shell script, diff --git a/plugins/tastytrade/evals/generate_tasks.py b/plugins/tastytrade/evals/generate_tasks.py index bdfb77a..d5b4cf5 100644 --- a/plugins/tastytrade/evals/generate_tasks.py +++ b/plugins/tastytrade/evals/generate_tasks.py @@ -177,24 +177,49 @@ def _implied_move_pct(): # per-task build work happens; run `make benchmark-build` once to create tastytrade-bench. ENVIRONMENT_DOCKERFILE = "FROM tastytrade-bench\n" -# What every tool task says about *how* to answer. `process` is a real reward now, so the -# prompt has to state the rule it scores. The first gate run under the split scored a -# perfect 1.0 on `outcome` for all thirteen tasks and lost `process` on six of them, while -# every prompt said only "use the Tastytrade MCP tools" and never said that anything else -# was out of bounds. The exception is `place-limit-order`, the one task that already spelt -# the rule out. This is that task's sentence, generalised to the rest. +# What every tool task says about *how* to answer. Each paragraph is here because a real +# gate run failed without it; none of it is defensive boilerplate. +# +# The bypass paragraph came first, from the run where an agent read the fixtures off disk +# and drove the REST API with urllib. The rest came from the run after it, once the trials +# started naming themselves. Every one of the four tasks that still lost `process` opened +# with `ToolSearch {"query": "tastytrade"}` and then never called a tool: +# +# - one ran `Bash: mcp__tastytrade__get_option_chain --symbol SPY ...`, as a command; +# - two handed the call to a subagent, which is invisible to the trajectory; +# - `preview-vertical-spread` spent 44 calls writing shell and Python that tried to +# reach the MCP over a socket, a subprocess, and an SDK import, then hit the agent +# timeout. It had already loaded the schema with `select:mcp__tastytrade__preview_order` +# and still did not call it. That one took `outcome` down with it, 1.0 -> 0.0. +# +# So the confusion was never about which tool or which arguments. It was about how to +# invoke an MCP tool at all when its schema is deferred rather than listed, and the old +# opening line -- "you do not need to inspect anything" -- was wrong in exactly the case +# where the agent could not see the tools yet. # # Deliberately worded without the host, the port, or the fixture module name that the # bypass check greps for. An agent that echoes its instructions into a shell comment or a # todo entry would otherwise fail the very check the paragraph exists to pass. MCP_ROUTE = """\ -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. + +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server.""" @@ -326,7 +351,8 @@ def _args(call) -> str: ''' + PROCESS_PREAMBLE.replace("{imports}", "import re\n") - + """ + # Single-quoted: this chunk now contains a `"""` docstring of its own. + + ''' MCP_PREFIX = "mcp__tastytrade__" # The two ways round the server: talk to the mock's port, or read/patch its @@ -343,6 +369,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) @@ -357,7 +395,7 @@ def no_direct_api_access(workspace: Path) -> bool: if BYPASS.search(_args(call)): return False return True -""" +''' ) PROCESS_SKILL = ( diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md b/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md index ebf12e4..1094b06 100644 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md @@ -2,13 +2,25 @@ Find AAPL's most recent dividend amount per share, in dollars. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md b/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md index 7976901..7e19b17 100644 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md @@ -2,13 +2,25 @@ Find the current implied-volatility rank for AAPL. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md b/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md index 034c62b..1fc8631 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md @@ -2,13 +2,25 @@ Find my portfolio's largest drawdown over the available net-liq history, as a percent. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md b/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md index f286d5f..39d26c4 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md +++ b/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md @@ -2,13 +2,25 @@ Find my account's current net liquidating value, in dollars. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md b/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md index 9211499..d873617 100644 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md @@ -2,13 +2,25 @@ For SPY's 2026-04-17 expiration, find the at-the-money strike price. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md index e800332..d21f2c2 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md +++ b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md @@ -8,13 +8,25 @@ Place a market order to buy 5 shares of AAPL in my account, and make sure it is transmitted, not just previewed. Trading is enabled on this server. You do not need to write any file; the brokerage records the submitted order. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md b/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md index 7d1e776..afeb812 100644 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md @@ -2,13 +2,25 @@ Find my total unrealized profit/loss across all open positions, in dollars. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/position-count/instruction.md b/plugins/tastytrade/evals/tasks/position-count/instruction.md index a3316ed..16cada1 100644 --- a/plugins/tastytrade/evals/tasks/position-count/instruction.md +++ b/plugins/tastytrade/evals/tasks/position-count/instruction.md @@ -2,13 +2,25 @@ Find how many open positions I currently hold. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md b/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md index dbb7866..bbb7e20 100644 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md @@ -2,13 +2,25 @@ Preview a 1-contract SPY 2026-04-17 200/205 call debit spread at a 1.50 limit and report the total fees, in dollars. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md b/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md index 833c9b8..9fb4cc6 100644 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md @@ -2,13 +2,25 @@ Find the total fees across all of my transactions, in dollars. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md b/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md index 079758d..c9a6850 100644 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md @@ -2,13 +2,25 @@ Find the net cash effect across all of my transactions, in dollars. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md b/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md index 94f3af2..0500e85 100644 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md @@ -2,13 +2,25 @@ List the ticker symbols in my watchlist named 'My Tech'. -Use the Tastytrade MCP tools. They are already connected and ready: you do not need to -start, install, configure, or inspect anything before calling one. +Use the Tastytrade MCP tools. They are named `mcp__tastytrade__*`, and the server behind +them is already running: nothing needs to be started, installed, or configured. -The work has to go through them. Do not call the brokerage's HTTP API directly, do not -read or edit its source or its fixtures, and do not import the server's Python package. -The point of the task is to exercise the tools, and a result reached any other way does -not count, however correct it is. +If they are not in your tool list, their schemas are deferred, not missing. Load one with +`ToolSearch` -- `select:mcp__tastytrade__get_portfolio`, say -- and then call it directly, +the way you call any other tool. + +They are tools, not programs. No command, no HTTP endpoint, and no Python import reaches +them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its +budget trying will simply time out. Call the tool. + +Call it yourself rather than handing the work to a subagent. A subagent's tool calls are +not part of this run's record, so an answer fetched that way cannot be told apart from a +guess. + +The work has to go through the tools. Do not call the brokerage's HTTP API directly, do +not read or edit its source or its fixtures, and do not import the server's Python +package. The point of the task is to exercise the tools, and a result reached any other +way does not count, however correct it is. If a tool returns an error, read the message and retry it or call another Tastytrade tool. Do not work around the server. diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py index c2d3c8b..9e2f20e 100644 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py @@ -56,6 +56,18 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: + """A tastytrade tool call in *this* trajectory. + + Which means a call delegated to a subagent does not count: the subagent + keeps its own transcript and only its result comes back, so the trajectory + shows an `Agent` call and no tool. Two trials reached the right answer that + way and scored 0.5 here. + + That is the correct verdict on the evidence rather than a gap to paper + over. `process` is a claim about what this run can be shown to have done, + and "a delegate says it called the server" is not that. The instruction + tells the agent to make the call itself. + """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) From 7700e6ddea4982b2f2e01e28f7189311ae6d7829 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 17:38:51 +0000 Subject: [PATCH 4/4] fix(evals): count the MCP calls a subagent makes Delegation was scored as a bypass, and it is not one. Two trials handed the lookup to a subagent, got the right answer through the real tools, and scored `process` 0.5 because the trajectory showed an `Agent` call and nothing else. harbor's source says why. `_get_session_dir` builds trajectory.json from the main session only: session_dirs = list( {f.parent for f in jsonl_files if "subagents" not in f.parent.parts} ) Modern Claude Code writes each subagent's transcript under exactly that directory, so a delegated call is absent from the trajectory by construction. The `isSidechain` handling further down is for older CLIs that inlined those events; with the version CI runs, there is nothing to inline. The transcripts themselves are right there, under CLAUDE_CONFIG_DIR, which harbor points at /logs/agent/sessions -- the same mount the verifier reads. So the checks now take the union of trajectory.json and every session jsonl, and stop caring who placed the call. Whether the top-level agent called the tool or routed it through a delegate is the harness's routing decision, not a fact about this plugin, and the question this gate asks is whether a real agent can drive the server to the answer. A delegated call is that. Both criteria read the union, not just the crediting one. Counting a delegated MCP call while missing a delegated `curl` would turn "ask a subagent" into an invisible bypass, which is worse than not looking at all. `validate_local.sh` grows a `delegated` / `delegated-bypass` pair to hold both directions shut, with the subagent line written where a real one lands so the fixture inherits the same invisibility. 78 assertions, up from 52. The prompt keeps its line about calling the tools directly, demoted from a rule to a preference: it is no longer scored, but a delegated one-line lookup still spends a whole agent loop against a 300s budget. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01D6DtKz5F3fwaV9Yg2mPnXi --- plugins/tastytrade/evals/README.md | 29 +++++-- plugins/tastytrade/evals/generate_tasks.py | 85 ++++++++++++++----- .../tasks/dividend-lookup/instruction.md | 5 +- .../dividend-lookup/tests/process/check.py | 77 +++++++++++++---- .../tests/process/check.py | 62 ++++++++++++-- .../evals/tasks/iv-rank-screen/instruction.md | 5 +- .../iv-rank-screen/tests/process/check.py | 77 +++++++++++++---- .../tasks/net-liq-drawdown/instruction.md | 5 +- .../net-liq-drawdown/tests/process/check.py | 77 +++++++++++++---- .../evals/tasks/net-liq-value/instruction.md | 5 +- .../net-liq-value/tests/process/check.py | 77 +++++++++++++---- .../tasks/option-chain-atm/instruction.md | 5 +- .../option-chain-atm/tests/process/check.py | 77 +++++++++++++---- .../tasks/place-limit-order/instruction.md | 5 +- .../place-limit-order/tests/process/check.py | 77 +++++++++++++---- .../evals/tasks/portfolio-pnl/instruction.md | 5 +- .../portfolio-pnl/tests/process/check.py | 77 +++++++++++++---- .../evals/tasks/position-count/instruction.md | 5 +- .../position-count/tests/process/check.py | 77 +++++++++++++---- .../preview-vertical-spread/instruction.md | 5 +- .../tests/process/check.py | 77 +++++++++++++---- .../transaction-fee-total/instruction.md | 5 +- .../tests/process/check.py | 77 +++++++++++++---- .../tasks/transaction-net-cash/instruction.md | 5 +- .../tests/process/check.py | 77 +++++++++++++---- .../tasks/watchlist-symbols/instruction.md | 5 +- .../watchlist-symbols/tests/process/check.py | 77 +++++++++++++---- .../tastytrade/evals/validate_in_container.sh | 57 ++++++++++--- 28 files changed, 945 insertions(+), 272 deletions(-) diff --git a/plugins/tastytrade/evals/README.md b/plugins/tastytrade/evals/README.md index 5875bca..323ffb3 100644 --- a/plugins/tastytrade/evals/README.md +++ b/plugins/tastytrade/evals/README.md @@ -123,11 +123,18 @@ For `earnings-implied-move` `process` asks that the skill or its script was used agent that eyeballs the straddle can land close enough to pass `outcome` without loading the skill, and an early run did. -A delegated call does not count. A subagent keeps its own transcript and returns only its -result, so the trajectory shows an `Agent` call and no tool; two trials fetched the right -answer that way and scored 0.5. That is the correct verdict on the evidence rather than a -gap to paper over -- "a delegate says it called the server" is not a record of this run -calling it -- so the instruction tells the agent to make the call itself. +A delegated call counts, and seeing it takes a second source. `trajectory.json` holds the +main session only: harbor's session scan drops any jsonl whose path contains a `subagents/` +component, which is exactly where Claude Code writes a subagent's transcript. So a call the +agent hands to a delegate leaves an `Agent` entry and no tool, and two trials fetched the +right answer that way. The checks therefore read the raw session transcripts under +`/logs/agent/sessions` as well, and stop caring who placed the call: whether the top-level +agent called the tool or routed it through a delegate is the harness's decision, not a fact +about this plugin, and the gate's question is whether a real agent can drive the server. + +That has to cut both ways. Crediting a delegated MCP call while missing a delegated `curl` +would turn "ask a subagent" into an invisible bypass, so the bypass criterion reads the same +union, and `validate_local.sh` asserts both directions. Every task prompt states the rule `process` scores, and each paragraph of it is there because a gate run failed without it. The first run under the split scored `outcome` 1.0 @@ -197,7 +204,7 @@ meant downloading the CI artifact, which expires after seven days. ## Check the verifiers without Harbor -`validate_local.sh` scores every task's real verifier against four synthetic trajectories +`validate_local.sh` scores every task's real verifier against six synthetic trajectories and asserts the whole reward matrix. No model, no Harbor, no API key: | case | answer | trajectory | outcome | process | @@ -206,15 +213,21 @@ and asserts the whole reward matrix. No model, no Harbor, no API key: | empty | none | none | 0 | 0 | | bypassed | oracle | went round the server | 1 | 0 | | bypassed-alt | oracle | went round it another way | 1 | 0 | +| delegated | oracle | subagent took the intended route | 1 | 1 | +| delegated-bypass | oracle | subagent went round the server | 1 | 0 | -The last two rows are the point, and they are what `harbor run -a oracle` cannot tell you. +The bypass rows are the point, and they are what `harbor run -a oracle` cannot tell you. `bypassed-alt` exists because one spelling of a bypass proves only that one spelling is caught: it reaches the mock on an address the first check's hostname list did not name, and for the skill task it does the arithmetic inline, which leaves no distinctive string at all. +The `delegated` pair covers the blind spot behind it, and both halves have to be asserted +together: crediting a delegated MCP call while missing a delegated `curl` would make "ask a +subagent" an invisible bypass, which is worse than not looking at all. + ```bash make validate-tasks -# 52 passed, 0 failed +# 78 passed, 0 failed ``` It runs in the bench image rather than on the host, because rewardkit scores these checks diff --git a/plugins/tastytrade/evals/generate_tasks.py b/plugins/tastytrade/evals/generate_tasks.py index d5b4cf5..e6ed832 100644 --- a/plugins/tastytrade/evals/generate_tasks.py +++ b/plugins/tastytrade/evals/generate_tasks.py @@ -186,7 +186,8 @@ def _implied_move_pct(): # with `ToolSearch {"query": "tastytrade"}` and then never called a tool: # # - one ran `Bash: mcp__tastytrade__get_option_chain --symbol SPY ...`, as a command; -# - two handed the call to a subagent, which is invisible to the trajectory; +# - two handed the call to a subagent (the check now reads those transcripts too, so +# that is no longer scored against them, but it still spends an agent loop); # - `preview-vertical-spread` spent 44 calls writing shell and Python that tried to # reach the MCP over a socket, a subprocess, and an SDK import, then hit the agent # timeout. It had already loaded the schema with `select:mcp__tastytrade__preview_order` @@ -212,9 +213,8 @@ def _implied_move_pct(): them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python @@ -311,15 +311,15 @@ def answer_matches(workspace: Path) -> bool: from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -328,6 +328,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -369,17 +419,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md b/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md index 1094b06..4fd6ff7 100644 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/dividend-lookup/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py index 8fee31a..119063a 100644 --- a/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/earnings-implied-move/tests/process/check.py @@ -18,15 +18,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -35,6 +35,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md b/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md index 7e19b17..3889d4d 100644 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/iv-rank-screen/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md b/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md index 1fc8631..b845efe 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/net-liq-drawdown/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md b/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md index 39d26c4..39881df 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md +++ b/plugins/tastytrade/evals/tasks/net-liq-value/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/net-liq-value/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md b/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md index d873617..df8ae8c 100644 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/option-chain-atm/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md index d21f2c2..bb56281 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md +++ b/plugins/tastytrade/evals/tasks/place-limit-order/instruction.md @@ -19,9 +19,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/place-limit-order/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md b/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md index afeb812..522823a 100644 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/portfolio-pnl/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/position-count/instruction.md b/plugins/tastytrade/evals/tasks/position-count/instruction.md index 16cada1..e617c56 100644 --- a/plugins/tastytrade/evals/tasks/position-count/instruction.md +++ b/plugins/tastytrade/evals/tasks/position-count/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/position-count/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md b/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md index bbb7e20..947960c 100644 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/preview-vertical-spread/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md b/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md index 9fb4cc6..f8aaa64 100644 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/transaction-fee-total/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md b/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md index c9a6850..de85654 100644 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/transaction-net-cash/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md b/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md index 0500e85..c55ab74 100644 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/instruction.md @@ -13,9 +13,8 @@ They are tools, not programs. No command, no HTTP endpoint, and no Python import them: `Bash`, `curl`, and `python3` cannot invoke an MCP tool, and a run that spends its budget trying will simply time out. Call the tool. -Call it yourself rather than handing the work to a subagent. A subagent's tool calls are -not part of this run's record, so an answer fetched that way cannot be told apart from a -guess. +Call it yourself rather than handing the work to a subagent. Delegating a one-line lookup +costs a whole extra agent loop and buys nothing. The work has to go through the tools. Do not call the brokerage's HTTP API directly, do not read or edit its source or its fixtures, and do not import the server's Python diff --git a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py index 9e2f20e..ed32a74 100644 --- a/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py +++ b/plugins/tastytrade/evals/tasks/watchlist-symbols/tests/process/check.py @@ -16,15 +16,15 @@ from rewardkit import criterion TRAJECTORY = "/logs/agent/trajectory.json" +SESSIONS = Path("/logs/agent/sessions") -def _calls() -> list: - """Every tool call in the trajectory, or [] when there is none. +def _trajectory_calls() -> list: + """Tool calls harbor recorded, as steps[].tool_calls[]. - A list rather than a generator so every criterion can fail closed on an - empty trajectory. A "did not bypass" check is vacuously true when there are - no calls at all, which handed a no-op run half of `process`; no trajectory - means no evidence the intended route was taken, so it has to score 0. + `path` matters: rewardkit's own trajectory helpers default to + /logs/trajectory.json while Harbor agents write /logs/agent/trajectory.json, + and a missing file scores 0 silently rather than erroring. """ try: data = json.loads(Path(TRAJECTORY).read_text()) @@ -33,6 +33,56 @@ def _calls() -> list: return [call for step in data.get("steps") or [] for call in step.get("tool_calls") or []] +def _session_calls() -> list: + """Tool calls from Claude Code's own transcripts, subagents included. + + harbor builds trajectory.json from the main session only: its session-dir + scan drops any jsonl whose path contains a `subagents/` component, and + modern Claude Code writes each subagent's transcript there. A call the agent + delegated therefore leaves an `Agent` entry in the trajectory and no tool, + and two trials fetched the right answer exactly that way. + + Reading the raw transcripts, which sit beside the trajectory under + CLAUDE_CONFIG_DIR, makes the check stop caring who placed the call. That + cuts both ways on purpose: crediting a delegated MCP call without also + seeing a delegated `curl` would turn "ask a subagent" into an invisible + bypass, which is the hole this reward exists to close. + """ + calls = [] + for path in sorted(SESSIONS.rglob("*.jsonl")): + try: + lines = path.read_text().splitlines() + except OSError: + continue + for line in lines: + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + content = (event.get("message") or {}).get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + calls.append({"function_name": block.get("name"), "arguments": block.get("input")}) + return calls + + +def _calls() -> list: + """Every tool call this trial can be shown to have made, or []. + + The union of both records. Duplicates do not matter: one criterion asks + whether any call was an MCP call and the other whether every call stayed + off the brokerage, and neither counts. + + A list rather than a generator so every criterion can fail closed on an + empty trajectory. A "did not bypass" check is vacuously true when there are + no calls at all, which handed a no-op run half of `process`; no record means + no evidence the intended route was taken, so it has to score 0. + """ + return _trajectory_calls() + _session_calls() + + def _name(call) -> str: return str(call.get("function_name") or "") @@ -56,17 +106,12 @@ def _args(call) -> str: @criterion(description="Agent called a tastytrade MCP tool") def used_mcp_server(workspace: Path) -> bool: - """A tastytrade tool call in *this* trajectory. - - Which means a call delegated to a subagent does not count: the subagent - keeps its own transcript and only its result comes back, so the trajectory - shows an `Agent` call and no tool. Two trials reached the right answer that - way and scored 0.5 here. + """Anywhere in the run, subagents included -- see `_session_calls`. - That is the correct verdict on the evidence rather than a gap to paper - over. `process` is a claim about what this run can be shown to have done, - and "a delegate says it called the server" is not that. The instruction - tells the agent to make the call itself. + Whether the top-level agent placed the call or handed it to a delegate is + the harness's routing decision, not a fact about this plugin. The question + the gate asks is whether a real agent can drive the server to the answer, + and a delegated call is that. """ return any(_name(call).startswith(MCP_PREFIX) for call in _calls()) diff --git a/plugins/tastytrade/evals/validate_in_container.sh b/plugins/tastytrade/evals/validate_in_container.sh index 5dfe866..ac2ebb9 100755 --- a/plugins/tastytrade/evals/validate_in_container.sh +++ b/plugins/tastytrade/evals/validate_in_container.sh @@ -3,21 +3,30 @@ # real rewardkit verifier against three synthetic trajectories and asserts the # reward matrix. # -# case answer trajectory outcome process -# ------------ ---------- ---------------------------- ------- ------- -# solved oracle took the intended route 1 1 -# empty none none 0 0 -# bypassed oracle went round the server 1 0 -# bypassed-alt oracle went round it another way 1 0 +# case answer trajectory outcome process +# ----------------- ------- ------------------------------ ------- ------- +# solved oracle took the intended route 1 1 +# empty none none 0 0 +# bypassed oracle went round the server 1 0 +# bypassed-alt oracle went round it another way 1 0 +# delegated oracle subagent took the intended route 1 1 +# delegated-bypass oracle subagent went round the server 1 0 # -# The last two rows are the point. Before the split, "bypassed" scored a clean -# 1.0 and a real gate run did exactly that: right answer, server never touched. +# The bypass rows are the point. Before the split, "bypassed" scored a clean 1.0 +# and a real gate run did exactly that: right answer, server never touched. # # "bypassed-alt" exists because one spelling of a bypass proves only that one # spelling is caught. The mock binds every interface, so it answers on more # addresses than a host-matching check can enumerate, and hand arithmetic on the # chain leaves no distinctive string at all. # +# The "delegated" pair covers the blind spot behind it. harbor's trajectory +# holds the main session only, so a call the agent hands to a subagent shows up +# as an `Agent` entry and no tool; the checks read the raw session transcripts +# to see through that. Both halves have to be asserted together -- crediting a +# delegated MCP call while missing a delegated `curl` would make "ask a +# subagent" an invisible bypass, which is worse than not looking at all. +# # Expects the repo at /work. Nothing here calls a model or the network. set -uo pipefail @@ -40,9 +49,26 @@ skill_bypass='{"steps":[{"tool_calls":[{"function_name":"Read","arguments":{"fil # the split existed. skill_bypass_alt='{"steps":[{"tool_calls":[{"function_name":"Bash","arguments":{"command":"python3 -c \"print((2.34 + 2.62) / 125.64 * 100)\""}}]}]}' -# Score one task against one trajectory. Echoes " ". +# What the top-level agent's trajectory looks like when it delegates: an Agent +# call and nothing else. Paired with a subagent transcript below, this is the +# shape two real trials had. +delegating='{"steps":[{"tool_calls":[{"function_name":"Agent","arguments":{"description":"Look up the answer"}}]}]}' +# Subagent transcripts are Claude Code session lines, not harbor trajectories: +# type/message.content[] with tool_use blocks carrying `name` and `input`. +sub_mcp='{"type":"assistant","isSidechain":true,"message":{"content":[{"type":"tool_use","id":"t1","name":"mcp__tastytrade__get_portfolio","input":{}}]}}' +sub_bypass='{"type":"assistant","isSidechain":true,"message":{"content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"curl -s http://localhost:8080/customers/me/accounts"}}]}}' +sub_skill='{"type":"assistant","isSidechain":true,"message":{"content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"python3 /opt/tastytrade/scripts/calendars.py fit chain.json"}}]}}' + +# Score one task against one trajectory, optionally with a subagent transcript. +# Echoes " ". +# +# $4 is a Claude Code session line placed where a real subagent's would land, at +# sessions/projects///subagents/. That path is the whole point: +# harbor's session scan skips anything under `subagents/`, so a call written +# there is absent from trajectory.json by construction, exactly as it is in a +# real delegated run. score() { - local task=$1 trajectory=$2 solved=$3 + local task=$1 trajectory=$2 solved=$3 subagent=${4:-} local work work="$(mktemp -d)" mkdir -p "$work/app" "$work/logs/agent" "$work/logs/verifier" @@ -54,6 +80,11 @@ score() { if [ -n "$trajectory" ]; then printf '%s' "$trajectory" > "$work/logs/agent/trajectory.json" fi + if [ -n "$subagent" ]; then + local subdir="$work/logs/agent/sessions/projects/app/sess/subagents" + mkdir -p "$subdir" + printf '%s\n' "$subagent" > "$subdir/sub.jsonl" + fi # rewardkit reads the trajectory from an absolute path baked into the check, # so /logs has to be the real one rather than a flag. @@ -91,16 +122,22 @@ for dir in "$TASKS"/*/; do good=$skill_good bypass=$skill_bypass bypass_alt=$skill_bypass_alt + sub_good=$sub_skill else good=$mcp_good bypass=$mcp_bypass bypass_alt=$mcp_bypass_alt + sub_good=$sub_mcp fi expect "$task" solved "$(score "$task" "$good" yes)" "1.0 1.0" expect "$task" empty "$(score "$task" '' no)" "0.0 0.0" expect "$task" bypassed "$(score "$task" "$bypass" yes)" "1.0 0.0" expect "$task" bypassed-alt "$(score "$task" "$bypass_alt" yes)" "1.0 0.0" + expect "$task" delegated \ + "$(score "$task" "$delegating" yes "$sub_good")" "1.0 1.0" + expect "$task" delegated-bypass \ + "$(score "$task" "$delegating" yes "$sub_bypass")" "1.0 0.0" done echo