fix(backtest): freeze cost book, rank on train, refuse live-go

docs/BACKTEST_AND_WFO.md

@@ -0,0 +1,55 @@
+# Canonical backtest and WFO
+
+Research replay lives in `src/backtest/*`. It is **not a live-go**. A passing
+WFO does not enable `trading_execution.enabled`, flip `test_mode`, or place
+orders. Paper → live is a separate human config + deploy step.
+
+## How to run
+
+Single window (historical simulator only):
+
+```bash
+uv run python scripts/run_backtest.py \
+  --symbol SOLUSDT --timeframe 1h \
+  --start 2024-01-01 --end 2024-06-01 \
+  --config config/settings.yaml \
+  --execution-profile execution_parity_v2
+```
+
+Walk-forward + gates (canonical WFO):
+
+```bash
+uv run python scripts/experiment_autopilot.py \
+  --config config/settings.yaml \
+  --symbol SOLUSDT --timeframe 1h \
+  --start 2024-01-01 --end 2026-01-01 \
+  --train-months 6 --test-months 3 \
+  --execution-profile execution_parity_v2
+```
+
+`--execution-profile execution_parity_v2` is closed-bar signal / next-open fill.
+`legacy_v1` fills at the signal-bar close and is kept only for old-run
+reproducibility. Autopilot defaults to v2.
+
+There is no `--live`, `--promote`, or `live_go` on these paths. Passing one
+is refused.
+
+## Clock
+
+Bar `time` is the **open**. A `1h` bar at `10:00` is `[10:00, 11:00)`.
+Strategies see completed OHLCV for that bar. v2 queues the fill for the next
+open. Unknown timeframe labels raise; they are not treated as 1 minute.
+
+## Cost book
+
+Frozen snapshot: `fee_rate` (commission / side of notional), `slippage_pct`
+(per-side price concession = spread + slip), `futures_funding_rate` (8h
+settlement fraction), `fixed_notional_usdt` (USDT size cap; 0 = uncapped).
+Defaults are 4 bps fee and 2 bps slip per side. Mutation after engine
+construction does not change fills.
+
+## Ranking
+
+Candidate ranking uses the **first train window** only. Holdout / WFO test
+metrics are reported, not used to order names. `scripts/run_wfo_sweep.py` is
+not a selection tool (it never applied `param_grid`).

docs/EXPERIMENT_AUTOPILOT.md

@@ -2,6 +2,9 @@
 
 `experiment_autopilot` combines baseline backtest, walk-forward validation, bootstrap uncertainty, and explicit acceptance gates in one command.
 
+This is **not a live-go**. See [`BACKTEST_AND_WFO.md`](BACKTEST_AND_WFO.md) for
+the clock, cost book, and paper→live boundary.
+
 For the higher-level RBI loop that decides when to run autoresearch, when to stop,
 and when a result can advance toward implementation or deployment, see
 [`RBI_AUTORESEARCH_LOOP.md`](RBI_AUTORESEARCH_LOOP.md).

docs/RESEARCH_FRAMEWORK.md

@@ -106,10 +106,9 @@ sweeping further.
 ### Tools
 
 ```bash
-python scripts/run_backtest.py           # single config backtest
-python scripts/run_full_backtest.py      # full parameter backtest
-python scripts/run_wfo.py               # walk-forward optimization
-python scripts/run_wfo_sweep.py          # WFO across param grid
+python scripts/run_backtest.py              # single-window simulator (not live)
+python scripts/experiment_autopilot.py      # canonical WFO + gates (not live)
+python scripts/run_config_search.py         # gated search; rank on train only
 ```
 
 ### Required tests (in order)

scripts/experiment_autopilot.py

@@ -36,6 +36,7 @@
     resolve_global_trend_filter,
 )
 from src.backtest.models import ExecutionProfile
+from src.backtest.research_safety import refuse_live_go
 from src.backtest.synthetic_eval import (  # noqa: E402
     SyntheticEvalResult,
     bars_from_range,
@@ -136,7 +137,9 @@ def parse_args() -> argparse.Namespace:
         "--synthetic-fit-end",
         help="Frozen ISO end for diagnostic regime fit",
     )
+    refuse_live_go(argv=sys.argv[1:])
     args = parser.parse_args()
+    refuse_live_go(flags=vars(args))
     if args.synthetic_diagnostic and (not args.synthetic_fit_start or not args.synthetic_fit_end):
         parser.error(
             "--synthetic-diagnostic requires --synthetic-fit-start and --synthetic-fit-end"

scripts/run_backtest.py

@@ -13,6 +13,7 @@
 from src.backtest.artifacts import create_manifest, git_revision, write_manifest
 from src.backtest.engine import BacktestEngine
 from src.backtest.factory import BacktestRequest, build_backtest_config
+from src.backtest.research_safety import refuse_live_go
 from src.db import close_pool, init_pool
 from src.features.reader import IndicatorReader
 from src.main import _resolve_strategy_config, load_settings
@@ -31,7 +32,7 @@ async def main():
         "--fee",
         type=float,
         default=None,
-        help="Trading fee rate override (defaults to 0.0004 futures / 0.001 spot)",
+        help="Trading fee rate override (fraction of notional per side; default 0.0004)",
     )
     parser.add_argument(
         "--quantity-step-size",
@@ -89,7 +90,9 @@ async def main():
         help="Execution semantics; legacy remains the default for reproducibility",
     )
 
+    refuse_live_go(argv=sys.argv[1:])
     args = parser.parse_args()
+    refuse_live_go(flags=vars(args))
 
     # Load settings from config file
     try:
@@ -118,7 +121,7 @@ async def main():
     futures_mode = bool(
         settings.futures and settings.futures.enabled and args.symbol in settings.futures.symbols
     )
-    fee_rate = args.fee if args.fee is not None else (0.0004 if futures_mode else 0.001)
+    fee_rate = args.fee
     min_notional_usdt = (
         args.min_notional if args.min_notional is not None else (20.0 if futures_mode else 0.0)
     )

scripts/run_config_search.py

@@ -13,15 +13,18 @@
 import sys
 import tempfile
 from dataclasses import asdict, dataclass
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 
 import yaml
 
 sys.path.append(os.getcwd())
 
 from src.backtest.engine import BacktestConfig, BacktestEngine, BacktestResult
+from src.backtest.experiment_autopilot import build_wfo_windows, wfo_inclusive_fetch_bounds
 from src.backtest.factory import BacktestRequest, build_backtest_config
+from src.backtest.ranking import RankedCandidate, rank_by_selection_score
+from src.backtest.research_safety import refuse_live_go
 from src.db import close_pool, get_pool, init_pool
 from src.features.reader import IndicatorReader
 from src.main import _resolve_strategy_config, load_settings
@@ -77,6 +80,8 @@ class CandidateMetrics:
     wfo_total_return_pct: float
     bootstrap_p_loss_pct: float
     profit_concentration_pct: float
+    selection_return_pct: float
+    selection_sharpe: float
     passes_gates: bool
     failure_reasons: str
 
@@ -561,7 +566,6 @@ def _build_backtest_config(
         strategy_classes=strategy_classes,
         strategy_configs=strategy_configs,
         aggregator_config=aggregator_config,
-        fee_rate=0.001,
     )
 
 
@@ -604,17 +608,13 @@ async def _run_wfo_windows(
     reader: IndicatorReader,
 ) -> tuple[int, float, float, float]:
     """Run rolling out-of-sample windows and return summary metrics."""
-    start_dt = datetime.fromisoformat(start)
-    end_dt = datetime.fromisoformat(end)
-    current = start_dt
+    windows = build_wfo_windows(start, end, train_months, test_months)
     window_returns: list[float] = []
     window_sharpes: list[float] = []
     window_trade_counts: list[int] = []
 
-    while current + timedelta(days=(train_months + test_months) * 30 + 1) < end_dt:
-        train_end = current + timedelta(days=train_months * 30)
-        test_start = train_end
-        test_end = min(test_start + timedelta(days=test_months * 30), end_dt)
+    for window in windows:
+        _, _, test_start, test_end = wfo_inclusive_fetch_bounds(window)
         cfg = _build_backtest_config(
             settings,
             strategy_classes,
@@ -623,15 +623,14 @@ async def _run_wfo_windows(
             raw_config,
             symbol,
             timeframe,
-            test_start.isoformat(),
-            test_end.isoformat(),
+            test_start,
+            test_end,
             apply_global_trend_filter,
         )
         result = await _run_backtest(cfg, reader)
         window_returns.append(result.total_return_pct)
         window_sharpes.append(result.sharpe_ratio)
         window_trade_counts.append(result.total_trades)
-        current = train_end
 
     if not window_returns:
         return 0, 0, 0.0, 0.0, 100.0
@@ -692,6 +691,27 @@ async def _evaluate_candidate(
             candidate.apply_global_trend_filter,
         )
         full_result = await _run_backtest(full_config, reader)
+        windows = build_wfo_windows(start, end, train_months, test_months)
+        if windows:
+            train_start, train_end, _, _ = wfo_inclusive_fetch_bounds(windows[0])
+            train_config = _build_backtest_config(
+                settings,
+                strategy_classes,
+                strategy_configs,
+                aggregator_config,
+                updated_raw_config,
+                symbol,
+                timeframe,
+                train_start,
+                train_end,
+                candidate.apply_global_trend_filter,
+            )
+            train_result = await _run_backtest(train_config, reader)
+            selection_return_pct = train_result.total_return_pct
+            selection_sharpe = train_result.sharpe_ratio
+        else:
+            selection_return_pct = 0.0
+            selection_sharpe = 0.0
         trade_returns = [trade.return_pct for trade in full_result.trades]
         bootstrap_p_loss_pct = _compute_bootstrap_loss_probability(
             trade_returns, bootstrap_iterations
@@ -751,6 +771,8 @@ async def _evaluate_candidate(
             wfo_total_return_pct=wfo_total_return_pct,
             bootstrap_p_loss_pct=bootstrap_p_loss_pct,
             profit_concentration_pct=profit_concentration_pct,
+            selection_return_pct=selection_return_pct,
+            selection_sharpe=selection_sharpe,
             passes_gates=not failure_reasons,
             failure_reasons=",".join(failure_reasons),
         )
@@ -784,7 +806,9 @@ def _write_artifacts(output_prefix: str, metrics: list[CandidateMetrics]) -> tup
 async def main() -> None:
     """Entry point."""
     configure_logger("WARNING")
+    refuse_live_go(argv=sys.argv[1:])
     args = parse_args()
+    refuse_live_go(flags=vars(args))
 
     config_path = Path(args.config)
     base_settings = load_settings(config_path)
@@ -861,22 +885,29 @@ async def main() -> None:
         await close_pool()
 
     passing = [metric for metric in metrics if metric.passes_gates]
-    ranking = sorted(
-        metrics,
-        key=lambda item: (
-            item.passes_gates,
-            item.wfo_total_return_pct,
-            item.wfo_mean_sharpe,
-            item.total_return_pct,
-            -item.bootstrap_p_loss_pct,
-        ),
-        reverse=True,
-    )
+    ranked_names = {
+        item.name: item
+        for item in rank_by_selection_score(
+            [
+                RankedCandidate(
+                    name=metric.name,
+                    selection_score=metric.selection_sharpe,
+                    holdout_score=metric.wfo_mean_sharpe,
+                )
+                for metric in metrics
+            ]
+        )
+    }
+    ranking = list(ranked_names.values())
+    metric_by_name = {metric.name: metric for metric in metrics}
 
-    print("\nTop candidates:")
-    for metric in ranking[:10]:
+    print("\nTop candidates (selection-window rank; holdout reported only):")
+    for ranked in ranking[:10]:
+        metric = metric_by_name[ranked.name]
         print(
             f"{metric.name}: pass={metric.passes_gates} "
+            f"sel_sharpe={metric.selection_sharpe:.2f} "
+            f"sel_return={metric.selection_return_pct:.2f}% "
             f"trades={metric.total_trades} "
             f"wfo_trades={metric.wfo_total_trades} "
             f"return={metric.total_return_pct:.2f}% "

scripts/run_full_backtest.py

@@ -8,6 +8,7 @@
 sys.path.append(os.getcwd())
 
 from src.backtest.engine import BacktestConfig, BacktestEngine
+from src.backtest.research_safety import refuse_live_go
 from src.db import close_pool, init_pool
 from src.features.reader import IndicatorReader
 from src.strategy.bollinger_strategy import BollingerBounceStrategy
@@ -16,6 +17,7 @@
 
 
 async def main():
+    refuse_live_go(argv=sys.argv[1:])
     parser = argparse.ArgumentParser(description="Run crypto strategy backtest")
     parser.add_argument("--symbol", type=str, required=True, help="Trading pair (e.g. BTCUSDT)")
     parser.add_argument("--timeframe", type=str, default="1m", help="Timeframe (e.g. 1m, 5m, 1h)")
@@ -25,6 +27,7 @@ async def main():
     parser.add_argument("--fee", type=float, default=0.001, help="Trading fee rate (0.001 = 0.1%%)")
 
     args = parser.parse_args()
+    refuse_live_go(flags=vars(args))
 
     db_config = {
         "host": os.getenv("DB_HOST", "localhost"),