From 788f6f66caa93c7d789f358afe84e06bacddc546 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 30 Aug 2026 21:40:32 +0000 Subject: [PATCH 1/3] fix(backtest): put leftover WFO runners on the canonical clock Advertised run_wfo.py still used 30-day months, inclusive test_end fetches, and legacy_v1 fills while the quality bar only covered autopilot/search. Wire it, the short-comparison script, and entry overlap to calendar windows, half-open fetch bounds, and execution_parity_v2. This is not a live-go. Co-Authored-By: Grok 4.6 Co-authored-by: Yderf --- CLAUDE.md | 2 +- docs/BACKTEST_AND_WFO.md | 10 ++-- docs/MATH_MODELS_ROADMAP.md | 2 +- docs/RESEARCH_FRAMEWORK.md | 11 +++- scripts/analyze_entry_overlap.py | 14 +++-- scripts/run_wfo.py | 82 +++++++++++++++++++---------- scripts/run_wfo_short_comparison.py | 66 ++++++++++++----------- tests/test_backtest_quality_bar.py | 40 +++++++++++++- 8 files changed, 156 insertions(+), 71 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3ae982fd..a21e9f0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -547,7 +547,7 @@ loss_limits: |--------|---------| | `scripts/run_backtest.py` | Run single backtest | | `scripts/run_full_backtest.py` | Full parameter backtest | -| `scripts/run_wfo.py` | Walk-forward optimization | +| `scripts/run_wfo.py` | Fixed-config WFO OOS (same clock as `experiment_autopilot`; not optimization) | | `scripts/smoke_test.py` | Quick connectivity check | | `scripts/migrate.py` | Apply database migrations | | `scripts/config_doctor.py` | Validate configuration | diff --git a/docs/BACKTEST_AND_WFO.md b/docs/BACKTEST_AND_WFO.md index 190944c1..01848b48 100644 --- a/docs/BACKTEST_AND_WFO.md +++ b/docs/BACKTEST_AND_WFO.md @@ -41,9 +41,13 @@ Strategies see completed OHLCV for that bar. v2 queues the fill for the next open. Unknown timeframe labels raise; they are not treated as 1 minute. WFO test windows are `[start, end)`. `fetch_range` uses `time >= start AND -time <= end`, so canonical autopilot (and the search CLIs) translate `end` -with `wfo_inclusive_fetch_bounds()` before the fetch. Frozen historical -artifacts stay as written; do not rerun them to “fix” old gates. +time <= end`, so canonical autopilot, the search CLIs, `scripts/run_wfo.py`, +`scripts/run_wfo_short_comparison.py`, and `scripts/analyze_entry_overlap.py` +translate `end` with `wfo_inclusive_fetch_bounds()` before the fetch. Frozen +historical artifacts stay as written; do not rerun them to “fix” old gates. + +`scripts/run_wfo.py` is a thin fixed-config OOS runner on that clock. It does +not optimize parameters. Gated WFO is `scripts/experiment_autopilot.py`. ## Cost book diff --git a/docs/MATH_MODELS_ROADMAP.md b/docs/MATH_MODELS_ROADMAP.md index f1e0559d..b188d951 100644 --- a/docs/MATH_MODELS_ROADMAP.md +++ b/docs/MATH_MODELS_ROADMAP.md @@ -44,7 +44,7 @@ surface is still unauthorized work. | Piece | What is already true | |-------|----------------------| -| Validation | WFO + bootstrap + concentration gates live in `src/backtest/experiment_autopilot.py` (`GateConfig`, `WfoWindow`). Drive with `scripts/experiment_autopilot.py` / `scripts/run_wfo.py`. | +| Validation | WFO + bootstrap + concentration gates live in `src/backtest/experiment_autopilot.py` (`GateConfig`, `WfoWindow`). Drive gated WFO with `scripts/experiment_autopilot.py`. `scripts/run_wfo.py` is a thin fixed-config OOS runner on the same calendar / half-open clock — not parameter optimization. | | Primary technical stack | `TrendPullbackStrategy` plus MTF (`mtf_breakout`, `mtf_continuation`, `multi_timeframe_regime`, `regime_router`). Production agents on this stack are **paper / disarmed**; several configs emit zero fills at current aggregator thresholds. | | Engine | `execution_parity_v2` evaluates at bar close and fills at next-bar open (`fill_source="next_bar_open"`). `legacy_v1` remains the signal-close compatibility path. The `scripts/experiment_autopilot.py` CLI defaults to `execution_parity_v2`. Futures qty is step-truncated in `src/backtest/sizing.py`. | | Costs | `src/backtest/cost_overrides.py` (`CostProfile`: `legacy` / `realistic` / `corrected`). Corrected book: fee `0.0004`, slip `0.0002`, `scaled_8h` funding. Model work must not edit cost overrides. | diff --git a/docs/RESEARCH_FRAMEWORK.md b/docs/RESEARCH_FRAMEWORK.md index 98fc8474..4caa6183 100644 --- a/docs/RESEARCH_FRAMEWORK.md +++ b/docs/RESEARCH_FRAMEWORK.md @@ -122,10 +122,17 @@ python scripts/run_config_search.py # gated search; rank on train only WFO is the primary validation tool. Fixed-window backtests are insufficient alone. ```bash -python scripts/run_wfo.py BTCUSDT 1h 2021-01-01 2022-01-01 --config +python scripts/experiment_autopilot.py \ + --config --symbol BTCUSDT --timeframe 1h \ + --start 2021-01-01 --end 2022-01-01 \ + --train-months 6 --test-months 3 \ + --execution-profile execution_parity_v2 +# Thin OOS-only runner (same calendar + half-open fetch; no gates): +# python scripts/run_wfo.py BTCUSDT 1h 2021-01-01 2022-01-01 --config ``` -Acceptable result: OOS Sharpe ≥ 0.6 across ≥ 3 folds. +Acceptable result: OOS Sharpe ≥ 0.6 across ≥ 3 folds. Use `experiment_autopilot` +for gated WFO. `run_wfo.py` is not parameter optimization. #### 3b. Parameter stability check diff --git a/scripts/analyze_entry_overlap.py b/scripts/analyze_entry_overlap.py index 14d76be1..a9a70c5e 100755 --- a/scripts/analyze_entry_overlap.py +++ b/scripts/analyze_entry_overlap.py @@ -25,7 +25,11 @@ _run_backtest, ) from scripts.run_autoresearch import _deep_merge, _read_yaml # noqa: E402 -from src.backtest.experiment_autopilot import build_wfo_windows # noqa: E402 +from src.backtest.experiment_autopilot import ( # noqa: E402 + build_wfo_windows, + wfo_inclusive_fetch_bounds, +) +from src.backtest.research_safety import refuse_live_go # noqa: E402 from src.db import close_pool, get_pool, init_pool # noqa: E402 from src.features.reader import IndicatorReader # noqa: E402 from src.main import _resolve_strategy_config, load_settings # noqa: E402 @@ -237,13 +241,14 @@ async def _collect_oos_entries( entries: list[datetime] = [] async with reader: for window in windows: + _, _, test_start, test_end = wfo_inclusive_fetch_bounds(window) window_config = _build_backtest_config( settings=settings, raw_config=raw_config, symbol=run_symbol, timeframe=run_timeframe, - start=window.test_start, - end=window.test_end, + start=test_start, + end=test_end, strategy_classes=strategy_classes, strategy_configs=strategy_configs, aggregator_config=aggregator_config, @@ -251,6 +256,7 @@ async def _collect_oos_entries( disable_trend_filter=False, replay_sentiment_path=spec.replay_sentiment_log, replay_sentiment_max_age_hours=spec.replay_sentiment_max_age_hours, + execution_profile="execution_parity_v2", ) result_bt = await _run_backtest(reader, window_config) for trade in result_bt.trades: @@ -388,7 +394,9 @@ def _interpret(report: dict[str, Any]) -> str: async def main() -> None: + refuse_live_go(argv=sys.argv[1:]) args = parse_args() + refuse_live_go(flags=vars(args)) configure_logger("INFO") manifest = _load_manifest(Path(args.manifest)) specs = _parse_agents(manifest) diff --git a/scripts/run_wfo.py b/scripts/run_wfo.py index e7417a5c..463f1ddb 100644 --- a/scripts/run_wfo.py +++ b/scripts/run_wfo.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 -"""Walk-Forward Optimization runner.""" +"""Fixed-config walk-forward OOS runner. + +This is not parameter optimization. Calendar windows, half-open inclusive +fetch bounds, and ``execution_parity_v2`` match +``scripts/experiment_autopilot.py``. Use that script for gated WFO. +Not a live-go. +""" + +from __future__ import annotations import argparse import asyncio @@ -7,13 +15,33 @@ import os import subprocess import sys -from datetime import datetime, timedelta from pathlib import Path from statistics import mean sys.path.append(os.getcwd()) -from src.backtest.research_safety import refuse_live_go +from src.backtest.experiment_autopilot import ( # noqa: E402 + WfoWindow, + build_wfo_windows, + wfo_inclusive_fetch_bounds, +) +from src.backtest.research_safety import refuse_live_go # noqa: E402 + +ExecutionProfile = str + + +def oos_fetch_windows( + start: str, + end: str, + train_months: int, + test_months: int, +) -> list[tuple[WfoWindow, str, str]]: + """Calendar WFO folds with inclusive reader bounds for the OOS test only.""" + rows: list[tuple[WfoWindow, str, str]] = [] + for window in build_wfo_windows(start, end, train_months, test_months): + _, _, test_start, test_end = wfo_inclusive_fetch_bounds(window) + rows.append((window, test_start, test_end)) + return rows def run_backtest( @@ -24,6 +52,7 @@ def run_backtest( config_path: str, replay_sentiment_log: str | None = None, replay_sentiment_max_age_hours: float | None = None, + execution_profile: ExecutionProfile = "execution_parity_v2", ) -> dict[str, float] | None: cmd = [ sys.executable, @@ -38,6 +67,8 @@ def run_backtest( end, "--config", config_path, + "--execution-profile", + execution_profile, ] if replay_sentiment_log: cmd.extend(["--replay-sentiment-log", replay_sentiment_log]) @@ -70,53 +101,41 @@ async def wfo( config_path: str = "config/settings.yaml", replay_sentiment_log: str | None = None, replay_sentiment_max_age_hours: float | None = None, + execution_profile: ExecutionProfile = "execution_parity_v2", ) -> list[dict[str, str | float]]: - start_dt = datetime.fromisoformat(start) - end_dt = datetime.fromisoformat(end) results: list[dict[str, str | float]] = [] - current = start_dt - while current + timedelta(days=test_months * 30 + 1) < end_dt: - train_end = current + timedelta(days=train_months * 30) - test_start = train_end - test_end = test_start + timedelta(days=test_months * 30) - - if test_end > end_dt: - break - - train_str = train_end.strftime("%Y-%m-%d") - test_str = test_end.strftime("%Y-%m-%d") - + for window, test_start, test_end in oos_fetch_windows(start, end, train_months, test_months): print( - f"Train: {current.strftime('%Y-%m')} - {train_str} | Test: {train_end.strftime('%Y-%m')} - {test_str}" + f"Train: {window.train_start} - {window.train_end} | " + f"Test: {window.test_start} - {window.test_end}" ) metrics = run_backtest( symbol, timeframe, - train_str, - test_str, + test_start, + test_end, config_path, replay_sentiment_log=replay_sentiment_log, replay_sentiment_max_age_hours=replay_sentiment_max_age_hours, + execution_profile=execution_profile, ) if metrics: results.append( { "symbol": symbol, "timeframe": timeframe, - "train_start_month": current.strftime("%Y-%m"), - "train_end_date": train_str, - "test_start_month": train_end.strftime("%Y-%m"), - "test_end_date": test_str, + "train_start_month": window.train_start[:7], + "train_end_date": window.train_end[:10], + "test_start_month": window.test_start[:7], + "test_end_date": window.test_end[:10], "trades": metrics.get("trades", 0.0), "win_rate": metrics.get("win_rate", 0.0), "sharpe": metrics.get("sharpe", 0.0), } ) - current = train_end - if results: sharpe_mean = mean(float(r["sharpe"]) for r in results) win_rate_mean = mean(float(r["win_rate"]) for r in results) @@ -146,7 +165,9 @@ async def wfo( def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Run walk-forward optimization") + parser = argparse.ArgumentParser( + description="Fixed-config walk-forward OOS (same clock as experiment_autopilot)" + ) parser.add_argument("symbol", nargs="?", default="ETHUSDT") parser.add_argument("timeframe", nargs="?", default="5m") parser.add_argument("start", nargs="?", default="2023-01-01") @@ -158,6 +179,12 @@ def parse_args() -> argparse.Namespace: "--config", default=os.getenv("SETTINGS_PATH", "config/settings.yaml"), ) + parser.add_argument( + "--execution-profile", + choices=("legacy_v1", "execution_parity_v2"), + default="execution_parity_v2", + help="Execution semantics; v2 is the canonical WFO default", + ) parser.add_argument( "--replay-sentiment-log", type=str, @@ -189,5 +216,6 @@ def parse_args() -> argparse.Namespace: config_path=args.config, replay_sentiment_log=args.replay_sentiment_log, replay_sentiment_max_age_hours=args.replay_sentiment_max_age_hours, + execution_profile=args.execution_profile, ) ) diff --git a/scripts/run_wfo_short_comparison.py b/scripts/run_wfo_short_comparison.py index f2d4d4d6..f764a0f6 100644 --- a/scripts/run_wfo_short_comparison.py +++ b/scripts/run_wfo_short_comparison.py @@ -1,21 +1,32 @@ #!/usr/bin/env python3 -"""Walk-Forward Optimization comparison: Long-Only vs Long+Short.""" +"""Fixed-config WFO comparison: long-only vs long+short. + +Uses the same calendar windows, half-open fetch bounds, and +``execution_parity_v2`` fills as ``scripts/experiment_autopilot.py``. +Not a live-go. +""" import os import subprocess import sys -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path import pandas as pd sys.path.append(os.getcwd()) +from src.backtest.experiment_autopilot import ( # noqa: E402 + build_wfo_windows, + wfo_inclusive_fetch_bounds, +) +from src.backtest.research_safety import refuse_live_go # noqa: E402 + def run_backtest(symbol, timeframe, start, end, allow_short=False, sl=0.02, tp=0.05): """Run a single backtest and return metrics.""" cmd = [ - "python", + sys.executable, "scripts/run_backtest.py", "--symbol", symbol, @@ -29,6 +40,8 @@ def run_backtest(symbol, timeframe, start, end, allow_short=False, sl=0.02, tp=0 str(sl), "--tp", str(tp), + "--execution-profile", + "execution_parity_v2", ] if allow_short: cmd.append("--allow-short") @@ -59,45 +72,34 @@ def run_backtest(symbol, timeframe, start, end, allow_short=False, sl=0.02, tp=0 def wfo_comparison(symbol, timeframe, start, end, train_months=6, test_months=3, sl=0.02, tp=0.05): """Run WFO comparing long-only vs long+short.""" - start_dt = datetime.fromisoformat(start) - end_dt = datetime.fromisoformat(end) - results_long = [] results_short = [] - current = start_dt - window = 0 + windows = build_wfo_windows(start, end, train_months, test_months) - while current + timedelta(days=test_months * 30 + 1) < end_dt: - train_end = current + timedelta(days=train_months * 30) - test_start = train_end - test_end = test_start + timedelta(days=test_months * 30) + for index, window in enumerate(windows, start=1): + _, _, test_start, test_end = wfo_inclusive_fetch_bounds(window) + train_str = window.train_end[:10] + test_str = window.test_end[:10] - if test_end > end_dt: - break - - train_str = train_end.strftime("%Y-%m-%d") - test_str = test_end.strftime("%Y-%m-%d") - - window += 1 print(f"\n{'=' * 60}") - print(f"Window {window}: Train to {train_str} | Test to {test_str}") + print(f"Window {index}: Train to {train_str} | Test to {test_str}") print(f"{'=' * 60}") # Long-only test - print(f"\n[LONG-ONLY] Testing {test_start.strftime('%Y-%m-%d')} to {test_str}...") + print(f"\n[LONG-ONLY] Testing {window.test_start[:10]} to {test_str}...") metrics_long = run_backtest( symbol, timeframe, - test_start.strftime("%Y-%m-%d"), - test_str, + test_start, + test_end, allow_short=False, sl=sl, tp=tp, ) if metrics_long: - metrics_long["window"] = window - metrics_long["test_period"] = f"{test_start.strftime('%Y-%m')}-{test_str}" + metrics_long["window"] = index + metrics_long["test_period"] = f"{window.test_start[:7]}-{test_str}" results_long.append(metrics_long) print( f" Trades: {metrics_long.get('trades', 0)}, Win Rate: {metrics_long.get('win_rate', 0):.1f}%, " @@ -105,27 +107,25 @@ def wfo_comparison(symbol, timeframe, start, end, train_months=6, test_months=3, ) # Long+Short test - print(f"\n[LONG+SHORT] Testing {test_start.strftime('%Y-%m-%d')} to {test_str}...") + print(f"\n[LONG+SHORT] Testing {window.test_start[:10]} to {test_str}...") metrics_short = run_backtest( symbol, timeframe, - test_start.strftime("%Y-%m-%d"), - test_str, + test_start, + test_end, allow_short=True, sl=sl, tp=tp, ) if metrics_short: - metrics_short["window"] = window - metrics_short["test_period"] = f"{test_start.strftime('%Y-%m')}-{test_str}" + metrics_short["window"] = index + metrics_short["test_period"] = f"{window.test_start[:7]}-{test_str}" results_short.append(metrics_short) print( f" Trades: {metrics_short.get('trades', 0)}, Win Rate: {metrics_short.get('win_rate', 0):.1f}%, " f"Sharpe: {metrics_short.get('sharpe', 0):.2f}, Return: {metrics_short.get('return_pct', 0):.2f}%" ) - current = train_end - # Summary print(f"\n{'=' * 60}") print("WFO COMPARISON SUMMARY") @@ -235,6 +235,7 @@ def wfo_comparison(symbol, timeframe, start, end, train_months=6, test_months=3, if __name__ == "__main__": import argparse + refuse_live_go(argv=sys.argv[1:]) parser = argparse.ArgumentParser(description="WFO comparison: Long-Only vs Long+Short") parser.add_argument("--symbol", type=str, default="SOLUSDT", help="Trading pair") parser.add_argument("--timeframe", type=str, default="4h", help="Timeframe") @@ -244,6 +245,7 @@ def wfo_comparison(symbol, timeframe, start, end, train_months=6, test_months=3, parser.add_argument("--tp", type=float, default=0.05, help="Take profit (e.g. 0.05 for 5%)") args = parser.parse_args() + refuse_live_go(flags=vars(args)) wfo_comparison( symbol=args.symbol, diff --git a/tests/test_backtest_quality_bar.py b/tests/test_backtest_quality_bar.py index ea816cc6..ba8480ad 100644 --- a/tests/test_backtest_quality_bar.py +++ b/tests/test_backtest_quality_bar.py @@ -10,6 +10,7 @@ import pytest +from scripts.run_wfo import oos_fetch_windows from src.backtest.artifacts import create_manifest, write_manifest from src.backtest.engine import BacktestConfig, BacktestEngine from src.backtest.experiment_autopilot import ( @@ -328,6 +329,7 @@ def test_canonical_research_scripts_cannot_place_live_orders() -> None: Path("scripts/experiment_autopilot.py"), Path("scripts/run_wfo.py"), Path("scripts/run_wfo_sweep.py"), + Path("scripts/run_wfo_short_comparison.py"), Path("scripts/run_config_search.py"), Path("scripts/run_mtf_search.py"), Path("src/backtest/engine.py"), @@ -422,10 +424,17 @@ def test_search_scripts_use_identical_calendar_wfo_boundaries() -> None: config_src = Path("scripts/run_config_search.py").read_text(encoding="utf-8") mtf_src = Path("scripts/run_mtf_search.py").read_text(encoding="utf-8") autopilot_src = Path("scripts/experiment_autopilot.py").read_text(encoding="utf-8") - for src in (config_src, mtf_src): + wfo_src = Path("scripts/run_wfo.py").read_text(encoding="utf-8") + short_src = Path("scripts/run_wfo_short_comparison.py").read_text(encoding="utf-8") + overlap_src = Path("scripts/analyze_entry_overlap.py").read_text(encoding="utf-8") + for src in (config_src, mtf_src, wfo_src, short_src): assert "build_wfo_windows(start, end, train_months, test_months)" in src assert "wfo_inclusive_fetch_bounds(" in src - assert "for window in windows:" in src + assert ( + "for window in windows:" in src + or "for index, window in enumerate(windows" in src + or "oos_fetch_windows(" in src + ) assert "timedelta(days=" not in src assert "months * 30" not in src assert "train_months * 30" not in src @@ -434,6 +443,12 @@ def test_search_scripts_use_identical_calendar_wfo_boundaries() -> None: assert " start=window.test_start," not in autopilot_src assert " start=test_start," in autopilot_src assert " end=test_end," in autopilot_src + assert "wfo_inclusive_fetch_bounds(window)" in overlap_src + assert "end=window.test_end," not in overlap_src + assert "start=window.test_start," not in overlap_src + assert 'execution_profile="execution_parity_v2"' in overlap_src + assert 'default="execution_parity_v2"' in wfo_src + assert "--execution-profile" in wfo_src windows = build_wfo_windows("2024-01-01", "2026-01-01", 6, 3) sequence = [(window.test_start, window.test_end) for window in windows] assert sequence @@ -443,6 +458,25 @@ def test_search_scripts_use_identical_calendar_wfo_boundaries() -> None: (window.test_start, window.test_end) for window in build_wfo_windows("2024-01-01", "2026-01-01", 6, 3) ] + advertised = oos_fetch_windows("2024-01-01", "2026-01-01", 6, 3) + assert [window.test_start for window, _, _ in advertised] == [start for start, _ in sequence] + assert [window.test_end for window, _, _ in advertised] == [end for _, end in sequence] + first_window, first_fetch_start, first_fetch_end = advertised[0] + _, _, bound_start, bound_end = wfo_inclusive_fetch_bounds(first_window) + assert first_fetch_start == bound_start + assert first_fetch_end == bound_end + assert first_fetch_end != first_window.test_end + + +def test_thirty_day_wfo_windows_diverge_from_calendar() -> None: + """The old advertised runner used months*30; that is not the canonical clock.""" + from datetime import datetime, timedelta + + windows = build_wfo_windows("2024-01-01", "2026-01-01", 6, 3) + current = datetime.fromisoformat("2024-01-01") + thirty_test_end = (current + timedelta(days=6 * 30) + timedelta(days=3 * 30)).date().isoformat() + assert windows[0].test_end.startswith("2024-10-01") + assert thirty_test_end != "2024-10-01" def test_leap_year_and_month_end_wfo_windows_remain_disjoint() -> None: @@ -760,6 +794,8 @@ async def test_engine_rejects_invalid_mtf_timeframe_before_fetch() -> None: "scripts/run_backtest.py", "scripts/experiment_autopilot.py", "scripts/run_wfo.py", + "scripts/run_wfo_short_comparison.py", + "scripts/analyze_entry_overlap.py", "scripts/run_config_search.py", "scripts/run_mtf_search.py", "scripts/run_full_backtest.py", From 322a1cfef6814416c221a9889d4312062a11fd44 Mon Sep 17 00:00:00 2001 From: yderf Date: Tue, 1 Sep 2026 10:06:57 -0500 Subject: [PATCH 2/3] fix(backtest): parse Sharpe Ratio from run_backtest stdout run_wfo.py matched Sharpe: and silently filled 0.0 into every fold. Parse the actual Sharpe Ratio: label and lock it with an e2e test. Co-Authored-By: Grok 4.6 --- scripts/run_wfo.py | 30 +++++++++++++++-------- tests/test_run_wfo.py | 57 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 tests/test_run_wfo.py diff --git a/scripts/run_wfo.py b/scripts/run_wfo.py index 463f1ddb..1b6d4926 100644 --- a/scripts/run_wfo.py +++ b/scripts/run_wfo.py @@ -30,6 +30,25 @@ ExecutionProfile = str +def parse_run_backtest_stdout(stdout: str) -> dict[str, float]: + """Parse metrics from ``scripts/run_backtest.py`` stdout labels. + + ``run_backtest.py`` emits ``Sharpe Ratio:``, not ``Sharpe:``. Matching the + short label silently drops Sharpe and later fills 0.0 into every WFO fold. + """ + metrics: dict[str, float] = {} + for line in stdout.splitlines(): + stripped = line.strip() + if stripped.startswith("Total Trades:"): + metrics["trades"] = float(int(stripped.split(":", 1)[1].strip())) + elif stripped.startswith("Win Rate:"): + pct = stripped.split(":", 1)[1].strip().rstrip("%") + metrics["win_rate"] = float(pct) / 100.0 + elif stripped.startswith("Sharpe Ratio:"): + metrics["sharpe"] = float(stripped.split(":", 1)[1].strip()) + return metrics + + def oos_fetch_windows( start: str, end: str, @@ -78,16 +97,7 @@ def run_backtest( if result.returncode != 0: print(f"Backtest failed: {result.stderr}") return None - lines = result.stdout.splitlines() - metrics: dict[str, float] = {} - for line in lines: - if "Total Trades:" in line: - metrics["trades"] = float(int(line.split(":")[1])) - elif "Win Rate:" in line: - metrics["win_rate"] = float(line.split(":")[1].strip("%")) / 100.0 - elif "Sharpe:" in line: - metrics["sharpe"] = float(line.split(":")[1]) - return metrics + return parse_run_backtest_stdout(result.stdout) async def wfo( diff --git a/tests/test_run_wfo.py b/tests/test_run_wfo.py new file mode 100644 index 00000000..1fb58bbd --- /dev/null +++ b/tests/test_run_wfo.py @@ -0,0 +1,57 @@ +"""WFO child-output parser must use scripts/run_backtest.py labels.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.run_wfo import parse_run_backtest_stdout + +BACKTEST_SCRIPT = Path("scripts/run_backtest.py") + + +def _actual_run_backtest_results_block(*, trades: int, win_rate: float, sharpe: float) -> str: + """Stdout block using the same print labels as scripts/run_backtest.py.""" + return ( + "\n" + "=" * 40 + "\n" + "BACKTEST RESULTS\n" + "=" * 40 + "\n" + f"Total Trades: {trades}\n" + "Blocked BUY (session router): 0\n" + "Blocked BUY (basis filter): 0\n" + "Blocked BUY (cross-venue dislocation): 0\n" + f"Win Rate: {win_rate:.2f}%\n" + "Total Return: $12.50 (1.25%)\n" + "Max Drawdown: 3.00%\n" + "Final Equity: $1012.50\n" + f"Sharpe Ratio: {sharpe:.2f}\n" + "Profit Factor: 1.50\n" + "=" * 40 + "\n" + ) + + +def test_run_backtest_emits_sharpe_ratio_label() -> None: + source = BACKTEST_SCRIPT.read_text(encoding="utf-8") + assert 'print(f"Sharpe Ratio: {result.sharpe_ratio:.2f}")' in source + assert 'print(f"Total Trades: {result.total_trades}")' in source + assert 'print(f"Win Rate: {result.win_rate:.2f}%")' in source + assert 'print(f"Sharpe: {result.sharpe_ratio:.2f}")' not in source + + +def test_wfo_parser_reads_actual_run_backtest_labels() -> None: + stdout = _actual_run_backtest_results_block(trades=3, win_rate=50.00, sharpe=1.25) + metrics = parse_run_backtest_stdout(stdout) + assert metrics == {"trades": 3.0, "win_rate": 0.5, "sharpe": 1.25} + + +def test_legacy_sharpe_label_would_drop_ratio_line() -> None: + """Reproduction: searching for 'Sharpe:' misses 'Sharpe Ratio:'.""" + stdout = _actual_run_backtest_results_block(trades=3, win_rate=50.00, sharpe=1.25) + legacy: dict[str, float] = {} + for line in stdout.splitlines(): + if "Total Trades:" in line: + legacy["trades"] = float(int(line.split(":")[1])) + elif "Win Rate:" in line: + legacy["win_rate"] = float(line.split(":")[1].strip("%")) / 100.0 + elif "Sharpe:" in line: + legacy["sharpe"] = float(line.split(":")[1]) + assert legacy == {"trades": 3.0, "win_rate": 0.5} + assert "sharpe" not in legacy + assert parse_run_backtest_stdout(stdout)["sharpe"] == 1.25 From b41d7e3b8c80706aab77cd4b0fe2a2c55388cffe Mon Sep 17 00:00:00 2001 From: yderf Date: Tue, 1 Sep 2026 10:11:39 -0500 Subject: [PATCH 3/3] chore(backtest): retrigger CI on current main Empty commit so pull_request.synchronize runs against 7dd22b1. Co-Authored-By: Grok 4.6