From fcd05b623ef482e5aa98683f87d2fb55cfe895d8 Mon Sep 17 00:00:00 2001 From: devam <56464719+devam2001@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:07:02 +1100 Subject: [PATCH] Paper vs replay reconciliation: the last go-live gate Replays the exact period a paper session ran (same strategy, pairs, balance, max open trades), matches trades by pair and entry time, and checks that paper results are close to the replay: 28+ days, 80%+ of trades in both, realised return within 2 points, fills within 0.3% per side. The return gap is split into matched-trade execution, paper-only and replay-only trades, and anything the replay can't reproduce (manual pause/resume/flatten, halts, code changed mid-session) is flagged. - ridgeline/reconcile.py: session discovery, trade matching, checks - panel/replay.py: data top-up, session replay, background jobs - panel: Paper vs replay card; the Research tab's last gate shows real results - scripts/reconcile.py: CLI for panel sessions and the Docker bot - panel sessions now record their pairs, balance, start and a code fingerprint in .meta.json (config.json's pair list changes monthly) --- .gitignore | 1 + README.md | 34 +++- RESEARCH.md | 4 + docs/OPERATIONS.md | 63 ++++++ panel/app.py | 50 ++++- panel/bot.py | 16 +- panel/replay.py | 92 +++++++++ panel/static/app.js | 131 ++++++++++++- panel/static/index.html | 16 ++ panel/static/style.css | 4 + ridgeline/reconcile.py | 419 ++++++++++++++++++++++++++++++++++++++++ scripts/reconcile.py | 82 ++++++++ tests/test_panel_api.py | 69 +++++++ tests/test_reconcile.py | 199 +++++++++++++++++++ 14 files changed, 1172 insertions(+), 8 deletions(-) create mode 100644 ridgeline/reconcile.py create mode 100644 scripts/reconcile.py create mode 100644 tests/test_reconcile.py diff --git a/.gitignore b/.gitignore index b4493fc..7e4ed46 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,6 @@ all_usdt.json reports/ user_data/audit/ user_data/risk_state.json +*.reconcile.json .pytest_cache/ .ruff_cache/ diff --git a/README.md b/README.md index 30ab7ff..355bcce 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,9 @@ panel cannot start a live bot at all. | `user_data/config.private.json` | API login, JWT secret, exchange keys - **never share/commit** (loaded automatically via `add_config_files`) | | `panel/` | control panel (FastAPI + plain JS) | | `scripts/research.py` | IS/OOS backtests, walk-forward, fee stress, bias checks, go-live gates | -| `ridgeline/` | shared package: risk limits (`risk.py`, `freqtrade_risk.py`), audit trail, tear-sheet reports, alerts | +| `ridgeline/` | shared package: risk limits (`risk.py`, `freqtrade_risk.py`), audit trail, tear-sheet reports, paper-vs-replay reconciliation, alerts | | `scripts/report.py`, `scripts/watchdog.py` | tear-sheet generator; health/risk watchdog with Telegram alerts | +| `scripts/reconcile.py` | the last go-live gate: compares a paper session with a replay of the same weeks | | `tests/`, `.github/workflows/ci.yml` | test suite (risk, audit, lookahead, universe, panel, hygiene) run by GitHub Actions | | `Dockerfile`, `docker-compose.yml` | 24/7 deployment: bot + monitor-mode panel + watchdog | | `docs/OPERATIONS.md` | runbook: risk limits, kill switch, alerts, VPS deployment, incidents, monthly routine | @@ -150,6 +151,36 @@ Then it must paper-trade for **4+ weeks** with results close to a replay of the same weeks. If nothing passes, the answer is "don't go live yet" - not "loosen the gates". +### Paper vs replay (the last gate) + +Use the panel's **Paper trading → Paper vs replay** card, or run: + +```bash +python scripts/reconcile.py +``` + +It replays the exact period the session ran, with the same strategy, pairs and +starting balance. Then it matches the trades one by one (same pair, entries +within 1.5 candles). The session is "close to the replay" when all of these hold: + +- it ran for at least 28 days +- at least 80% of trades happened in both +- realised return is within 2 percentage points of the replay +- fills cost at most 0.3% per side compared with the replay's candle prices + +The result also splits the return gap into three parts: +- trades in both, which shows execution differences +- paper-only trades +- replay-only trades + +It flags anything the replay can't reproduce: +- manual pauses, resumes and flattens +- drawdown halts +- strategy code or parameters that changed mid-session + +The Research tab shows the newest result for each strategy. +See [docs/OPERATIONS.md](docs/OPERATIONS.md#paper-vs-replay). + Plain backtests still work as usual: ```bash @@ -229,6 +260,7 @@ The tests cover: - audit tamper detection - a lookahead check: signals must not change when future candles are removed - point-in-time universe selection +- paper-vs-replay trade matching, gap breakdown and checks - the panel's dry-run guard and kill-switch sequence - repo hygiene: no secrets, and no live config in git diff --git a/RESEARCH.md b/RESEARCH.md index dff261a..bba2fae 100644 --- a/RESEARCH.md +++ b/RESEARCH.md @@ -113,6 +113,10 @@ The gap between the IS results is the most important finding: ## Next steps 1. **Paper-trade TrendRider4h for 4+ weeks** from the panel. Use Reset wallet and pick TrendRider4h. `config.json` now holds today's point-in-time top 25. + - Then run **Paper vs replay** on that session, or + `python scripts/reconcile.py `. + - That's the last gate: the paper results must be close to a replay of + the same weeks. 2. **Re-run `research.py gates` monthly.** New data adds OOS trades, and the 100-trade gate will be settled honestly. 3. **Optionally re-hyperopt TrendRider4h** on point-in-time IS universes. Treat diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 9301d58..55a795c 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -10,6 +10,7 @@ How to run ridgeline 24/7, watch it, and act when something goes wrong. - [Deploying on a VPS (Docker)](#deploying-on-a-vps-docker) - [Incident checklist](#incident-checklist) - [Monthly routine](#monthly-routine) +- [Paper vs replay](#paper-vs-replay) - [Reports](#reports) --- @@ -161,11 +162,73 @@ git pull && docker compose up -d --build python scripts/research.py universe --download --write-config # refresh data + live pair list python scripts/research.py gates # re-test every strategy on new OOS data python scripts/report.py --db user_data/paper/.sqlite --month YYYY-MM +python scripts/reconcile.py # paper vs replay of the same weeks ``` If you change the pair list or the strategy, restart the bot: `docker compose restart bot`, or Reset in the panel. +## Paper vs replay + +The last go-live gate: 4+ weeks of paper trading whose results are close to a +replay (backtest) of the same weeks. Run it from **Paper trading → Paper vs +replay** in the panel, or from the command line: + +```bash +python scripts/reconcile.py # a panel session +python scripts/reconcile.py --db user_data/tradesv3.docker.sqlite --log user_data/logs/bot.log # the Docker bot +``` + +**What it does.** +1. Tops up candle data for the session's pairs. +2. Replays the session's period, starting at the candle its first decision + used and using the same strategy, pairs, starting balance and max open trades. +3. Matches trades: same pair, entries at most 1.5 candles apart. + +The result is saved next to the session's database as +`.reconcile.json`. The Research tab shows the newest result for each +strategy. + +**Where the settings come from.** A session started from the panel records +its strategy, pair list, balance, start time and a fingerprint of the +strategy code in `.meta.json`. That matters because `config.json`'s +pair list changes monthly. Older sessions and the Docker bot fall back to the +bot's log and database. `--strategy`, `--pairs`, `--balance`, `--start` and +`--end` override any of them. + +**Checks.** All must pass: + +| Check | Target | Why | +|---|---|---| +| Length | ≥ 28 days | Long enough to see a few trades and some market change | +| Trades in both | ≥ 80% | Signals fire the same way live as in the backtest | +| Return gap | ≤ 2 percentage points | The backtest's results are reachable | +| Fill cost | ≤ 0.3% per side | Fills land near the candle prices the backtest assumes | + +Fill cost is the average difference between paper and replay prices. It's +positive when paper paid more on entry or received less on exit. Exits only +count when both sides left for the same reason. + +**Reading a gap.** The return gap is split into three parts: +- **trades in both**: execution, meaning fills, stop timing and exit prices +- **paper-only trades** +- **replay-only trades** + +One-sided trades usually come from: +- an entry order that didn't fill within its 10-minute timeout +- a different equity, so a risk limit or the open-trade cap blocked one side +- signals right at the start or end of the period + +The result notes things the replay can't reproduce, so check those first: +- manual pause, resume or flatten (from the audit trail) +- drawdown halts, which need a manual resume live but resume automatically in backtests +- strategy code or parameters changed during the session +- paper trades newer than the latest closed candle + +**If it fails.** Don't loosen the targets. Find out why the two diverged. If +the backtest assumptions are wrong (for example, real fills are consistently +worse), fix the research model and re-run the gates before trying again. + ## Reports Two ways to get a tear sheet: diff --git a/panel/app.py b/panel/app.py index 89d202f..fae8ff7 100644 --- a/panel/app.py +++ b/panel/app.py @@ -12,8 +12,9 @@ from panel.bot import MONITOR_MODE, PaperBot, PaperBotError from panel.common import USERDIR, list_strategies, load_json -from panel.replay import REPLAY_DIR, ReplayJobs +from panel.replay import REPLAY_DIR, BacktestError, ReplayJobs, SessionReplays from ridgeline import audit +from ridgeline.reconcile import session_info from ridgeline.report import build_tearsheet, load_trades_backtest, load_trades_db, session_balance STATIC = Path(__file__).parent / "static" @@ -23,6 +24,7 @@ bot = PaperBot() replays = ReplayJobs() +session_replays = SessionReplays() class PaperStart(BaseModel): @@ -149,11 +151,30 @@ def _valid_month(month: Optional[str]) -> Optional[str]: return month or None -@app.get("/api/report", response_class=HTMLResponse) -def paper_report(session: str, month: Optional[str] = None): - db = USERDIR / "paper" / f"{_valid_id(session)}.sqlite" +def _session_db(session_id: str) -> Path: + db = USERDIR / "paper" / f"{_valid_id(session_id)}.sqlite" if not db.exists(): raise HTTPException(404, "No such paper session") + return db + + +@app.post("/api/paper/sessions/{session_id}/reconcile") +def reconcile_start(session_id: str): + db = _session_db(session_id) + try: + return session_replays.submit(session_info(db)) + except (ValueError, BacktestError) as e: + raise HTTPException(409, str(e)) + + +@app.get("/api/paper/sessions/{session_id}/reconcile") +def reconcile_get(session_id: str): + return session_replays.get(session_id, _session_db(session_id)) or {"status": "none"} + + +@app.get("/api/report", response_class=HTMLResponse) +def paper_report(session: str, month: Optional[str] = None): + db = _session_db(session) try: trades = load_trades_db(db) return build_tearsheet(trades, session_balance(db), title="Ridgeline - paper trading report", @@ -202,6 +223,25 @@ def replay_history(): return replays.history() +def _latest_reconciliations() -> dict: + """Newest paper-vs-replay result per strategy (panel sessions and the + Docker bot's database), for the last go-live gate.""" + out = {} + for path in [*(USERDIR / "paper").glob("*.reconcile.json"), *USERDIR.glob("*.reconcile.json")]: + r = load_json(path, None) or {} + if r.get("status") != "done": + continue + strategy = r["session"]["strategy"] + if strategy not in out or r["generated"] > out[strategy]["generated"]: + out[strategy] = {"session_id": r["session"]["session_id"], "generated": r["generated"], + "days": r["period"]["days"], "close_to_replay": r["close_to_replay"], + "passed": sum(c["pass"] for c in r["checks"]), "checks": len(r["checks"])} + return out + + @app.get("/api/research") def research(): - return load_json(USERDIR / "research" / "latest.json", None) or {} + report = load_json(USERDIR / "research" / "latest.json", None) or {} + if report: + report["paper_checks"] = _latest_reconciliations() + return report diff --git a/panel/bot.py b/panel/bot.py index dbcf5e3..5f232ca 100644 --- a/panel/bot.py +++ b/panel/bot.py @@ -29,6 +29,7 @@ from panel.common import FREQTRADE, ROOT, USERDIR, config_args, load_json, merged_config from ridgeline import audit +from ridgeline.reconcile import strategy_fingerprint from ridgeline.risk import RiskLimits, build_state, limits_summary, load_halt, set_halt MONITOR_MODE = os.environ.get("PANEL_MODE", "").lower() == "monitor" @@ -110,6 +111,18 @@ def start(self, balance: float, strategy: str) -> dict: }, indent=2)) args = [*config_args(), "-c", str(session_cfg)] self._verify_dry_run(args) + # What this session runs, for reconciling it with a replay later: + # config.json's pair list changes monthly, and code can change mid-session. + cfg = merged_config() + (PAPER_DIR / f"{session_id}.meta.json").write_text(json.dumps({ + "session_id": session_id, + "strategy": strategy, + "balance": balance, + "pairs": cfg.get("exchange", {}).get("pair_whitelist", []), + "max_open_trades": cfg.get("max_open_trades"), + "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "code": strategy_fingerprint(strategy), + }, indent=2)) log_path = PAPER_DIR / f"{session_id}.log" cmd = [*FREQTRADE, "trade", *args, "--strategy", strategy, "--logfile", str(log_path)] @@ -305,6 +318,7 @@ def sessions(self) -> list[dict]: rows = [] for db in sorted(PAPER_DIR.glob("*.sqlite"), reverse=True): cfg = load_json(db.with_suffix(".config.json"), {}) or {} + meta = load_json(db.with_suffix(".meta.json"), {}) or {} rows.append({"session_id": db.stem, "balance": cfg.get("dry_run_wallet"), - "size_kb": round(db.stat().st_size / 1024, 1)}) + "strategy": meta.get("strategy"), "size_kb": round(db.stat().st_size / 1024, 1)}) return rows diff --git a/panel/replay.py b/panel/replay.py index 1846300..0c57f1c 100644 --- a/panel/replay.py +++ b/panel/replay.py @@ -1,6 +1,8 @@ # Fast historical replay: run a Freqtrade backtest with a chosen starting # balance and date range, then reduce the result to what the UI and the # research scripts need (metrics, equity curve vs buy-and-hold, trades). +# Also replays a paper session's exact period to reconcile it against paper +# trading (ridgeline/reconcile.py has the comparison itself). import json import re @@ -15,8 +17,12 @@ import pandas as pd from panel.common import DATADIR, FREQTRADE, USERDIR, config_args +from ridgeline import audit +from ridgeline.reconcile import SessionInfo, load_paper_trades, load_replay_trades, reconcile, strategy_timeframe REPLAY_DIR = USERDIR / "replays" +# Daily EMA200 filters need ~200 days of history before the first candle. +WARMUP_DAYS = 250 class BacktestError(RuntimeError): @@ -67,6 +73,92 @@ def run_backtest( return summarize(outdir, strategy) +def download_data(pairs: list[str], since: pd.Timestamp, log_path: Path, + timeframes: tuple[str, ...] = ("1h", "4h", "1d")) -> None: + """Top up candle data for these pairs to the latest closed candle.""" + cmd = [*FREQTRADE, "download-data", *config_args(), "--pairs", *pairs, + "--timeframes", *timeframes, "--timerange", f"{since:%Y%m%d}-"] + with open(log_path, "w", encoding="utf-8", errors="replace") as log: + proc = subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT) + if proc.returncode != 0: + tail = log_path.read_text(encoding="utf-8", errors="replace").splitlines()[-15:] + raise BacktestError("Data download failed:\n" + "\n".join(tail)) + + +def replay_window(info: SessionInfo) -> tuple[str, pd.Timestamp]: + """The strategy's timeframe and the replay's first candle: the one the + session's first decision was based on. Raises if it hasn't closed yet.""" + timeframe = strategy_timeframe(info.strategy) + start = info.start.floor(pd.Timedelta(timeframe)) + first_close = start + pd.Timedelta(timeframe) + if first_close > pd.Timestamp.now(tz="UTC"): + raise BacktestError(f"Nothing to replay yet: the session's first {timeframe} candle closes at " + f"{first_close:%Y-%m-%d %H:%M} UTC.") + return timeframe, start + + +def replay_session(info: SessionInfo, outdir: Optional[Path] = None, download: bool = True) -> dict: + """Replay a paper session's exact period (same strategy, pairs, balance) + and reconcile it with what paper trading did.""" + timeframe, start = replay_window(info) + outdir = Path(outdir or REPLAY_DIR / f"session-{info.session_id}") + outdir.mkdir(parents=True, exist_ok=True) + if download: + # BTC/USDT feeds the strategies' market-regime filter even when it isn't traded. + download_data(sorted({*info.pairs, "BTC/USDT"}), start - pd.Timedelta(days=WARMUP_DAYS), + outdir / "download.log") + run_backtest(info.strategy, f"{int(start.timestamp())}-{int(info.end.timestamp())}", info.balance, + pairs=info.pairs, max_open_trades=info.max_open_trades, outdir=outdir) + replay, meta = load_replay_trades(outdir) + return reconcile(load_paper_trades(info.db), replay, info=info, replay_end=meta["end"], + timeframe=timeframe, events=audit.tail(100_000)) + + +def reconcile_path(db: Path) -> Path: + """Where a session's latest reconciliation is kept: next to its database.""" + return Path(db).with_suffix(".reconcile.json") + + +class SessionReplays: + """Paper-vs-replay reconciliations for the panel, one at a time. The result + is written next to the session's database, so it outlives the panel.""" + + def __init__(self): + self._pool = ThreadPoolExecutor(max_workers=1) + self._lock = threading.Lock() + self._running: dict[str, dict] = {} + + def submit(self, info: SessionInfo) -> dict: + replay_window(info) # fail fast if there's nothing to replay yet + with self._lock: + if info.session_id in self._running: + return self._running[info.session_id] + job = {"status": "running", "session_id": info.session_id, + "started": datetime.now(timezone.utc).isoformat(timespec="seconds")} + self._running[info.session_id] = job + self._pool.submit(self._run, info) + return job + + def _run(self, info: SessionInfo) -> None: + try: + result = replay_session(info) + except Exception as e: # surfaced to the UI as-is + result = {"status": "error", "error": str(e), "session": info.to_dict(), + "generated": datetime.now(timezone.utc).isoformat(timespec="seconds")} + try: + reconcile_path(info.db).write_text(json.dumps(result, indent=2, default=str)) + finally: + with self._lock: + self._running.pop(info.session_id, None) + + def get(self, session_id: str, db: Path) -> Optional[dict]: + with self._lock: + if session_id in self._running: + return self._running[session_id] + path = reconcile_path(db) + return json.loads(path.read_text()) if path.exists() else None + + def summarize(outdir: Path, strategy: str) -> dict: from freqtrade.data.btanalysis import load_backtest_stats diff --git a/panel/static/app.js b/panel/static/app.js index c2a0330..f7c4099 100644 --- a/panel/static/app.js +++ b/panel/static/app.js @@ -108,6 +108,10 @@ async function refreshPaper() { } else if (session) { rows.push(["Strategy", esc(session.strategy)], ["Starting balance", fmtMoney(session.balance)]); } + if (session && session.session_id !== knownSessionId) { + knownSessionId = session.session_id; + loadReconSessions(knownSessionId); + } if (session) { rows.push([st.managed ? "Session" : "Last session", `${esc(session.session_id)}`]); rows.push(["Report", `Performance tear sheet ↗`]); @@ -198,6 +202,124 @@ $("#paper-reset").addEventListener("click", () => { paperAction(() => api("/api/paper/reset", { method: "POST", body: { balance: bal, strategy: $("#paper-strategy").value } })); }); +// ===================================================================== +// Paper vs replay +// ===================================================================== +let knownSessionId = null; +let reconPolling = null; + +const checkMark = (pass) => `${pass ? "✓" : "✗"}`; +const fmtPts = (v) => (v == null || isNaN(v) ? "–" : `${v > 0 ? "+" : ""}${Number(v).toFixed(2)} pts`); + +async function loadReconSessions(preferred) { + const sel = $("#recon-session"); + const rows = await api("/api/paper/sessions").catch(() => []); + const want = preferred || sel.value; + sel.innerHTML = rows.map((s) => ``).join(""); + if (want && rows.some((s) => s.session_id === want)) sel.value = want; + $("#recon-run").disabled = !rows.length; + if (!rows.length) { + $("#recon-status").textContent = "No paper sessions yet."; + return; + } + loadRecon(); +} + +async function loadRecon() { + const id = $("#recon-session").value; + if (!id) return; + const r = await api(`/api/paper/sessions/${encodeURIComponent(id)}/reconcile`) + .catch((e) => ({ status: "error", error: e.message })); + if (id !== $("#recon-session").value) return; // selection changed meanwhile + renderRecon(r); + if (r.status === "running") pollRecon(id); +} + +async function pollRecon(id) { + if (reconPolling === id) return; + reconPolling = id; + while (reconPolling === id && $("#recon-session").value === id) { + await new Promise((res) => setTimeout(res, 2000)); + const r = await api(`/api/paper/sessions/${encodeURIComponent(id)}/reconcile`).catch(() => null); + if (!r || $("#recon-session").value !== id) break; + renderRecon(r); + if (r.status !== "running") break; + } + if (reconPolling === id) reconPolling = null; +} + +function renderRecon(r) { + const status = $("#recon-status"); + const verdict = $("#recon-verdict"); + const result = $("#recon-result"); + $("#recon-run").disabled = r.status === "running"; + verdict.hidden = r.status !== "done"; + result.hidden = r.status !== "done"; + if (r.status === "none") { + status.textContent = "Not compared yet."; + return; + } + if (r.status === "running") { + status.textContent = "Replaying… (tops up candle data first; usually under a minute)"; + return; + } + if (r.status === "error") { + status.innerHTML = `Comparison failed: ${esc(r.error).slice(0, 600)}`; + return; + } + + verdict.className = `pill ${r.close_to_replay ? "on" : "ext"}`; + verdict.textContent = r.close_to_replay ? "Close to replay" : "Not close yet"; + status.textContent = `Compared ${r.generated.replace("T", " ").slice(0, 16)} UTC · ` + + `${r.period.start.slice(0, 16).replace("T", " ")} → ${r.period.end.slice(0, 16).replace("T", " ")} UTC (${r.period.days} days)`; + + $("#recon-checks").innerHTML = r.checks.map((c) => + `
  • ${checkMark(c.pass)}${esc(c.label)} (${c.value == null ? "n/a" : esc(c.value)})
    • `).join(""); + + const p = r.paper, rp = r.replay, m = r.matching, g = r.gap, f = r.fills; + const total = m.matched + m.paper_only + m.replay_only; + $("#recon-tiles").innerHTML = [ + tile("Paper return (realised)", fmtPct(p.return_pct), signClass(p.return_pct), `${p.closed} closed, ${p.open} open`), + tile("Replay return (realised)", fmtPct(rp.return_pct), signClass(rp.return_pct), `${rp.closed} closed, ${rp.open} open`), + tile("Gap (paper − replay)", fmtPts(g.return_gap_pp), signClass(g.return_gap_pp), + `matched ${fmtMoney(g.matched_abs, "")} · paper-only ${fmtMoney(g.paper_only_abs, "")} · replay-only ${fmtMoney(g.replay_only_abs, "")}`), + tile("Trades in both", `${m.matched} / ${total}`, "", `${m.paper_only} paper-only, ${m.replay_only} replay-only`), + tile("Fill cost per side", fmtPct(f.fill_cost_pct, 3), f.fill_cost_pct > 0 ? "neg" : "", + `entry ${fmtPct(f.entry_cost_pct, 3)} · exit ${fmtPct(f.exit_cost_pct, 3)}`), + ].join(""); + + $("#recon-warnings").innerHTML = r.warnings.map((w) => `
  • ${esc(w)}
    • `).join(""); + + const label = { matched: "Both", paper_only: "Paper only", replay_only: "Replay only" }; + table($("#recon-trades"), [ + { label: "Pair", key: "pair" }, + { label: "In", render: (t) => esc(label[t.status]), cls: (t) => (t.status === "matched" ? "" : "neg") }, + { label: "Paper entry (UTC)", render: (t) => esc(t.paper_open ?? "–") }, + { label: "Replay entry (UTC)", render: (t) => esc(t.replay_open ?? "–") }, + { label: "Entry cost", num: true, render: (t) => fmtPct(t.entry_cost_pct, 3), cls: (t) => signClass(-t.entry_cost_pct) }, + { label: "Exit (paper / replay)", render: (t) => `${esc(t.paper_exit ?? "–")} / ${esc(t.replay_exit ?? "–")}` }, + { label: "Exit cost", num: true, render: (t) => fmtPct(t.exit_cost_pct, 3), cls: (t) => signClass(-t.exit_cost_pct) }, + { label: "Paper P&L", num: true, render: (t) => fmtPct(t.paper_profit_pct), cls: (t) => signClass(t.paper_profit_pct) }, + { label: "Replay P&L", num: true, render: (t) => fmtPct(t.replay_profit_pct), cls: (t) => signClass(t.replay_profit_pct) }, + ], r.trades, { empty: "No trades on either side in this period." }); +} + +$("#recon-session").addEventListener("change", () => loadRecon()); +$("#recon-run").addEventListener("click", async () => { + const id = $("#recon-session").value; + if (!id) return; + $("#recon-run").disabled = true; + $("#recon-status").textContent = "Starting…"; + try { + renderRecon(await api(`/api/paper/sessions/${encodeURIComponent(id)}/reconcile`, { method: "POST" })); + pollRecon(id); + } catch (e) { + $("#recon-status").innerHTML = `${esc(e.message)}`; + $("#recon-run").disabled = false; + } +}); + // ===================================================================== // Replay // ===================================================================== @@ -353,6 +475,12 @@ async function loadResearch() { const entries = Object.entries(r.strategies).sort((a, b) => b[1].oos.calmar - a[1].oos.calmar); for (const [name, s] of entries) { const passed = s.gates.filter((g) => g.pass).length; + const pc = (r.paper_checks || {})[name]; + const paperGate = pc + ? `
  • ${checkMark(pc.close_to_replay)}4+ weeks of paper trading close to replay ` + + `(session ${esc(pc.session_id)}: ${pc.days} days, ${pc.passed}/${pc.checks} checks)
    • ` + : `
  • ○4+ weeks of paper trading close to replay ` + + `(not compared yet: Paper trading → Paper vs replay)
    • `; const badge = s.eligible_for_paper ? `Eligible for 4-week paper test` : `${passed}/${s.gates.length} gates passed`; @@ -366,7 +494,7 @@ async function loadResearch() {
      ${s.gates.map((g) => `
    • ${g.pass ? "✓" : "✗"}${esc(g.label)} (${esc(g.value)})
      • `).join("")} -
    • ○4+ weeks of paper trading close to replay (manual)
      • + ${paperGate}
    @@ -464,6 +592,7 @@ $("#risk-flatten").addEventListener("click", () => { `

    Monitor mode: the bot runs as a Docker service, so Start/Stop/Reset are disabled here. Risk controls still work.

    `); } await loadStrategies().catch(() => {}); + loadReconSessions(); let tab = "paper"; try { tab = localStorage.getItem("tab") || "paper"; } catch {} document.querySelector(`.tab[data-tab="${tab}"]`)?.click(); diff --git a/panel/static/index.html b/panel/static/index.html index 243d908..e6d86fb 100644 --- a/panel/static/index.html +++ b/panel/static/index.html @@ -80,6 +80,22 @@

    Open trades

    Closed trades (latest 50)

    +
    +

    Paper vs replay

    +

    Replays the exact period a session ran (same strategy, pairs and balance) and matches + the trades one by one. The last go-live gate needs 4+ weeks of paper trading with every check passing.

    +
    + + + +
    + +

    Audit log

    Every control action and risk event, hash-chained so edits or deletions are detectable.

    diff --git a/panel/static/style.css b/panel/static/style.css index 869fcce..5d69db6 100644 --- a/panel/static/style.css +++ b/panel/static/style.css @@ -158,6 +158,10 @@ tr.clickable:hover td { background: var(--surface-2); } .gate-list { list-style: none; padding: 0; margin: 0 0 12px; display: grid; gap: 4px; } .gate-list li { display: flex; gap: 8px; } .gate-list .mark { width: 16px; font-weight: 700; } +.recon-controls { margin: 12px 0; } +.recon-controls select { max-width: 100%; } +.notes { margin: 0 0 14px; padding-left: 18px; color: var(--warn); font-size: 12px; display: grid; gap: 4px; } +.notes:empty { display: none; } .wf-bars { display: flex; gap: 3px; align-items: center; height: 60px; margin-top: 6px; } .wf-bar { flex: 1; min-width: 6px; border-radius: 2px; } diff --git a/ridgeline/reconcile.py b/ridgeline/reconcile.py new file mode 100644 index 0000000..30dcf22 --- /dev/null +++ b/ridgeline/reconcile.py @@ -0,0 +1,419 @@ +"""Paper vs replay reconciliation. + +The last go-live gate: a strategy paper-trades for 4+ weeks, and the paper +results must be close to a replay (backtest) of the same weeks, run with the +same strategy, pairs and starting balance. Passing it means the backtests the +research relies on hold up in near-live execution: + - signals fire on time + - fills land near the candle prices the backtest assumes + - stops and exits behave the same way + + session_info() what a paper session ran: strategy, pairs, balance, start, end + load_paper_trades() / load_replay_trades() + match_trades() pair trades up by pair and entry time + reconcile() summary, return-gap breakdown, checks and a verdict + +Running the replay itself (data download + backtest) is replay_session() in +panel/replay.py; scripts/reconcile.py and the panel both call it. +""" + +import ast +import hashlib +import json +import re +import sqlite3 +from collections import Counter +from contextlib import closing +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import pandas as pd + +from ridgeline import ROOT, USERDIR + +STRATEGY_DIR = USERDIR / "strategies" + +MIN_DAYS = 28 +# Paper and replay entries for the same signal can be up to a candle apart: +# a session that starts mid-candle acts on the last closed candle at once, +# and an unfilled entry order is retried on the next candle. +MATCH_TOLERANCE_CANDLES = 1.5 + +CHECKS = [ + # (key, label, comparison, target) + ("days", f"Paper-traded for at least {MIN_DAYS} days", ">=", MIN_DAYS), + ("match_rate_pct", "At least 80% of trades happened in both", ">=", 80), + ("return_gap_pp", "Realised return within 2 points of the replay", "<=", 2.0), + ("fill_cost_pct", "Fills cost at most 0.3% per side vs the replay", "<=", 0.3), +] + +# Audit events the replay can't reproduce: manual actions, and drawdown halts +# (a human resumes them live; the backtest resumes after a fixed review period). +UNMODELLED_EVENTS = {"entries_paused", "trading_resumed", "flatten", "risk_halt"} + +TRADE_COLUMNS = ["pair", "open_date", "close_date", "open_rate", "close_rate", "stake", + "profit_ratio", "profit_abs", "exit_reason", "is_open"] +CLOSE_COLUMNS = ["close_date", "close_rate", "profit_ratio", "profit_abs", "exit_reason"] + +_WHITELIST_RE = re.compile(r"Whitelist with \d+ pairs: (\[[^\]]*\])") +_STRATEGY_RE = re.compile(r"Using resolved strategy (\w+)") +_TIMEFRAME_RE = re.compile(r"^\s+timeframe\s*=\s*[\"'](\w+)[\"']", re.M) + + +# ---------------------------------------------------------------- session + +@dataclass +class SessionInfo: + session_id: str + db: Path + strategy: str + pairs: list[str] + balance: float + start: pd.Timestamp + end: pd.Timestamp + max_open_trades: Optional[int] = None + warnings: list[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return {"session_id": self.session_id, "strategy": self.strategy, "pairs": self.pairs, + "balance": self.balance, "start": self.start.isoformat(), "end": self.end.isoformat(), + "max_open_trades": self.max_open_trades} + + +def strategy_fingerprint(strategy: str, strategy_dir: Path = STRATEGY_DIR) -> dict[str, Optional[str]]: + """Hashes of everything that decides a strategy's trades: its code, its + hyperopt parameters and the shared risk layer.""" + files = [Path(strategy_dir) / f"{strategy}.py", Path(strategy_dir) / f"{strategy}.json", + ROOT / "ridgeline" / "risk.py", ROOT / "ridgeline" / "freqtrade_risk.py"] + return {p.name: hashlib.sha256(p.read_bytes()).hexdigest()[:16] if p.exists() else None for p in files} + + +def strategy_timeframe(strategy: str, strategy_dir: Path = STRATEGY_DIR) -> str: + match = _TIMEFRAME_RE.search((Path(strategy_dir) / f"{strategy}.py").read_text(encoding="utf-8")) + if not match: + raise ValueError(f"Couldn't find the timeframe of {strategy}") + return match.group(1) + + +def session_info(db: Path, *, log: Optional[Path] = None, strategy: Optional[str] = None, + pairs: Optional[list[str]] = None, balance: Optional[float] = None, + start: Optional[pd.Timestamp] = None, end: Optional[pd.Timestamp] = None) -> SessionInfo: + """What a session ran, from the best source available. Explicit arguments win. + After them, in order: + - .meta.json, written by the panel when a session starts + - the bot's log (log, or .log, with its rotated backups) + - the database itself + """ + db = Path(db) + meta = _load_json(db.with_suffix(".meta.json")) + logs = _log_files(Path(log) if log else db.with_suffix(".log")) + warnings = [] + + strategy = strategy or meta.get("strategy") or _first_match(logs, _STRATEGY_RE) or _db_strategy(db) + if not strategy: + raise ValueError(f"Couldn't tell which strategy {db.name} ran. Pass it explicitly.") + + if not pairs: + pairs = meta.get("pairs") + if not pairs: + lists = [ast.literal_eval(m) for m in _all_matches(logs, _WHITELIST_RE)] + if not lists: + raise ValueError(f"Couldn't tell which pairs {db.name} traded (no meta file and no whitelist " + "in its log). Pass them explicitly.") + pairs = lists[0] + if any(set(other) != set(pairs) for other in lists[1:]): + warnings.append("The pair list changed during the session (a restart with a new config?). " + "The replay uses the first one.") + + if balance is None: + balance = meta.get("balance") or _load_json(db.with_suffix(".config.json")).get("dry_run_wallet") + if balance is None: + balance = _load_json(USERDIR / "config.json").get("dry_run_wallet", 1000) + warnings.append(f"Starting balance not recorded for this session; assuming {balance} from config.json.") + + start = _utc(start) or _utc(meta.get("started")) or _db_start(db) + if start is None: + raise ValueError(f"Couldn't tell when {db.name} started. Pass the start explicitly.") + if end is None: + # The bot logs a heartbeat every minute, so the newest log write is when + # the session was last alive (about now, if it's still running). + if logs: + end = pd.Timestamp(max(p.stat().st_mtime for p in logs), unit="s", tz="UTC") + else: + end = pd.Timestamp.now(tz="UTC") + warnings.append("No bot log found, so the session is assumed to still be running.") + end = _utc(end) + + recorded = meta.get("code") + if recorded: + current = strategy_fingerprint(strategy) + changed = [name for name, digest in recorded.items() if current.get(name) != digest] + if changed: + warnings.append(f"Changed since the session started: {', '.join(changed)}. " + "The replay runs the current version, so some differences are expected.") + + return SessionInfo(session_id=db.stem, db=db, strategy=strategy, pairs=list(pairs), balance=float(balance), + start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings) + + +def _load_json(path: Path) -> dict: + try: + data = json.loads(Path(path).read_text()) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + +def _utc(value) -> Optional[pd.Timestamp]: + if value is None or value == "": + return None + ts = pd.Timestamp(value) + return ts.tz_localize("UTC") if ts.tzinfo is None else ts.tz_convert("UTC") + + +def _log_files(log: Path) -> list[Path]: + """The log and its rotated backups (log.1 is newer than log.2), oldest first.""" + backups = sorted(log.parent.glob(log.name + ".*"), key=lambda p: -int(p.suffix[1:]) if p.suffix[1:].isdigit() else 0) + return [p for p in [*backups, log] if p.is_file()] + + +def _all_matches(logs: list[Path], regex: re.Pattern) -> list[str]: + return [m.group(1) for p in logs for m in regex.finditer(p.read_text(encoding="utf-8", errors="replace"))] + + +def _first_match(logs: list[Path], regex: re.Pattern) -> Optional[str]: + for p in logs: + m = regex.search(p.read_text(encoding="utf-8", errors="replace")) + if m: + return m.group(1) + return None + + +def _db_query(db: Path, sql: str): + if not Path(db).exists(): + return None + with closing(sqlite3.connect(f"file:{Path(db).as_posix()}?mode=ro", uri=True)) as con: + try: + row = con.execute(sql).fetchone() + except sqlite3.Error: + return None + return row[0] if row else None + + +def _db_strategy(db: Path) -> Optional[str]: + return _db_query(db, "SELECT strategy FROM trades WHERE strategy IS NOT NULL LIMIT 1") + + +def _db_start(db: Path) -> Optional[pd.Timestamp]: + # Freqtrade records its first start in the database (naive UTC). + return _utc(_db_query(db, "SELECT datetime_value FROM KeyValueStore WHERE key = 'bot_start_time'")) + + +# ---------------------------------------------------------------- trades + +def load_paper_trades(db: Path) -> pd.DataFrame: + with closing(sqlite3.connect(f"file:{Path(db).as_posix()}?mode=ro", uri=True)) as con: + df = pd.read_sql_query( + "SELECT pair, open_date, close_date, open_rate, close_rate, stake_amount AS stake, " + "close_profit AS profit_ratio, close_profit_abs AS profit_abs, exit_reason, is_open FROM trades", con) + return _normalise(df) + + +def load_replay_trades(directory: Path) -> tuple[pd.DataFrame, dict]: + """Trades from a backtest result directory, plus its start, end and timeframe. + Trades the backtest force-closed at its last candle are still open in + reality, so they come back open, the same way paper shows them.""" + from freqtrade.data.btanalysis import load_backtest_stats + + strat = next(iter(load_backtest_stats(directory)["strategy"].values())) + df = pd.DataFrame(strat["trades"]).reindex(columns=[*TRADE_COLUMNS, "stake_amount"]) + df["stake"] = df.pop("stake_amount") + df = _normalise(df) + end = pd.Timestamp(strat["backtest_end"], tz="UTC") + left_open = (df["exit_reason"] == "force_exit") & (df["close_date"] >= end) + df.loc[left_open, CLOSE_COLUMNS] = None + df.loc[left_open, "is_open"] = True + return df, {"start": pd.Timestamp(strat["backtest_start"], tz="UTC"), "end": end, + "timeframe": strat["timeframe"], "starting_balance": float(strat["starting_balance"])} + + +def _normalise(df: pd.DataFrame) -> pd.DataFrame: + df = df.copy() + for col in ("open_date", "close_date"): + df[col] = pd.to_datetime(df[col], utc=True, errors="coerce") + for col in ("open_rate", "close_rate", "stake", "profit_ratio", "profit_abs"): + df[col] = pd.to_numeric(df[col], errors="coerce") + df["exit_reason"] = df["exit_reason"].astype(object) + df["is_open"] = df["is_open"].fillna(False).astype(bool) + return df[TRADE_COLUMNS].sort_values("open_date").reset_index(drop=True) + + +def match_trades(paper: pd.DataFrame, replay: pd.DataFrame, + tolerance: pd.Timedelta) -> tuple[list[tuple[int, int]], list[int], list[int]]: + """Pair each paper trade with the replay trade on the same pair whose entry + is closest in time, within `tolerance`. Closest pairs are matched first. + Returns (matched index pairs, paper-only indexes, replay-only indexes).""" + candidates = [] + for i, p in paper.iterrows(): + same_pair = replay[replay["pair"] == p["pair"]] + for j, opened in same_pair["open_date"].items(): + gap = abs(p["open_date"] - opened) + if gap <= tolerance: + candidates.append((gap, i, j)) + matched, used_p, used_r = [], set(), set() + for _, i, j in sorted(candidates): + if i not in used_p and j not in used_r: + matched.append((i, j)) + used_p.add(i) + used_r.add(j) + return (sorted(matched), [i for i in paper.index if i not in used_p], + [j for j in replay.index if j not in used_r]) + + +# ---------------------------------------------------------------- compare + +def _check(op: str, value, target) -> bool: + if value is None: + return False + return {">=": value >= target, "<=": value <= target}[op] + + +def _realised(t) -> float: + return 0.0 if t["is_open"] or pd.isna(t["profit_abs"]) else float(t["profit_abs"]) + + +def _r(value, digits: int = 2): + return None if value is None or pd.isna(value) else round(float(value), digits) + + +def _when(ts) -> Optional[str]: + return None if pd.isna(ts) else f"{ts:%Y-%m-%d %H:%M}" + + +def reconcile(paper: pd.DataFrame, replay: pd.DataFrame, *, info: SessionInfo, replay_end: pd.Timestamp, + timeframe: str, events: Optional[list[dict]] = None) -> dict: + """Compare a paper session's trades with a replay of the same period. + + `replay_end` is the open time of the replay's last candle. Paper trades + opened after that candle closes can't be in the replay yet: they're + left out, and paper trades closed after it count as still open. + Fill costs are positive when paper did worse than the replay: it paid more + on entry or received less on exit. + """ + tf = pd.Timedelta(timeframe) + start = info.start.floor(tf) + cutoff = min(info.end, replay_end + tf) + paper, replay = _normalise(paper), _normalise(replay) + + too_recent = paper["open_date"] >= cutoff + paper = paper[~too_recent].copy() + late = paper["close_date"] >= cutoff + paper.loc[late, CLOSE_COLUMNS] = None + paper.loc[late, "is_open"] = True + + matched, paper_only, replay_only = match_trades(paper, replay, tf * MATCH_TOLERANCE_CANDLES) + + rows, entry_costs, exit_costs = [], [], [] + both_closed = same_exit = 0 + gap_matched = 0.0 + for i, j in matched: + p, r = paper.loc[i], replay.loc[j] + entry_cost = (p["open_rate"] / r["open_rate"] - 1) * 100 + entry_costs.append(entry_cost) + exit_cost = None + if not p["is_open"] and not r["is_open"]: + both_closed += 1 + # Exit prices are only comparable when both sides left for the same reason. + if p["exit_reason"] == r["exit_reason"]: + same_exit += 1 + exit_cost = (1 - p["close_rate"] / r["close_rate"]) * 100 + exit_costs.append(exit_cost) + gap_matched += _realised(p) - _realised(r) + rows.append(_row("matched", p, r, entry_cost, exit_cost)) + rows += [_row("paper_only", paper.loc[i], None) for i in paper_only] + rows += [_row("replay_only", None, replay.loc[j]) for j in replay_only] + rows.sort(key=lambda row: row["paper_open"] or row["replay_open"]) + + def side(df: pd.DataFrame) -> dict: + closed = df[~df["is_open"]] + realised = float(closed["profit_abs"].sum()) + return {"trades": len(df), "closed": len(closed), "open": int(df["is_open"].sum()), + "realised_abs": round(realised, 2), "return_pct": round(realised / info.balance * 100, 2)} + + paper_side, replay_side = side(paper), side(replay) + total = len(matched) + len(paper_only) + len(replay_only) + costs = entry_costs + exit_costs + gap = paper_side["return_pct"] - replay_side["return_pct"] + values = { + "days": round((cutoff - info.start).total_seconds() / 86400, 1), + "match_rate_pct": round(100 * len(matched) / total, 1) if total else None, + "return_gap_pp": round(abs(gap), 2), + "fill_cost_pct": _r(sum(costs) / len(costs), 3) if costs else None, + } + checks = [{"key": key, "label": label, "value": values[key], "pass": _check(op, values[key], target)} + for key, label, op, target in CHECKS] + + warnings = list(info.warnings) + if too_recent.any(): + warnings.append(f"{int(too_recent.sum())} paper trade(s) opened after the replay's last candle " + f"({_when(cutoff)} UTC) and aren't compared yet.") + if info.end - (replay_end + tf) > tf: + warnings.append(f"The replay's data stops at {_when(replay_end + tf)} UTC, before the session's last " + f"activity ({_when(info.end)} UTC). Download fresh data and run it again.") + if not total: + warnings.append("No trades on either side yet, so there's nothing to compare.") + unmodelled = [e for e in events or [] if e.get("event") in UNMODELLED_EVENTS + and info.start <= pd.Timestamp(e["ts"]) <= info.end] + if unmodelled: + listed = ", ".join(f"{name} x{n}" for name, n in Counter(e["event"] for e in unmodelled).items()) + warnings.append(f"During the session: {listed}. The replay can't reproduce manual actions or " + "the manual resume after a halt, so expect differences around them.") + + return { + "status": "done", + "generated": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "session": info.to_dict(), + "timeframe": timeframe, + "period": {"start": start.isoformat(), "end": cutoff.isoformat(), "days": values["days"]}, + "paper": paper_side, + "replay": replay_side, + "matching": {"matched": len(matched), "paper_only": len(paper_only), "replay_only": len(replay_only), + "match_rate_pct": values["match_rate_pct"], + "tolerance_hours": tf.total_seconds() * MATCH_TOLERANCE_CANDLES / 3600}, + "fills": { + "entry_cost_pct": _r(sum(entry_costs) / len(entry_costs), 3) if entry_costs else None, + "exit_cost_pct": _r(sum(exit_costs) / len(exit_costs), 3) if exit_costs else None, + "fill_cost_pct": values["fill_cost_pct"], + "same_exit_reason_pct": round(100 * same_exit / both_closed, 1) if both_closed else None, + }, + # Paper minus replay, in stake currency: the sum of the three parts is the total. + "gap": {"total_abs": round(paper_side["realised_abs"] - replay_side["realised_abs"], 2), + "matched_abs": round(gap_matched, 2), + "paper_only_abs": round(sum(_realised(paper.loc[i]) for i in paper_only), 2), + "replay_only_abs": round(-sum(_realised(replay.loc[j]) for j in replay_only), 2), + "return_gap_pp": round(gap, 2)}, + "checks": checks, + "close_to_replay": all(c["pass"] for c in checks), + "warnings": warnings, + "trades": rows, + } + + +def _row(status: str, p, r, entry_cost=None, exit_cost=None) -> dict: + """One line of the trade-by-trade comparison; p or r is None for a one-sided trade.""" + def side(t) -> dict: + if t is None: + return {"open": None, "entry": None, "exit": None, "profit_pct": None} + return {"open": _when(t["open_date"]), "entry": _r(t["open_rate"], 8), + "exit": "open" if t["is_open"] else t["exit_reason"], "profit_pct": _r(t["profit_ratio"] * 100)} + + ps, rs = side(p), side(r) + return { + "pair": (p if p is not None else r)["pair"], "status": status, + "paper_open": ps["open"], "replay_open": rs["open"], + "paper_entry": ps["entry"], "replay_entry": rs["entry"], "entry_cost_pct": _r(entry_cost, 3), + "paper_exit": ps["exit"], "replay_exit": rs["exit"], "exit_cost_pct": _r(exit_cost, 3), + "paper_profit_pct": ps["profit_pct"], "replay_profit_pct": rs["profit_pct"], + } diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 0000000..e220011 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,82 @@ +"""Compare a paper-trading session with a replay of the same weeks. + +The last go-live gate: 4+ weeks of paper trading whose results are close to +a replay (backtest) of the same period, with the same strategy, pairs and +balance. This tops up candle data, runs that replay, matches the trades and +checks how close they are. + +Usage (from the project root, with the venv Python): + python scripts/reconcile.py 20261011-010946 a panel session in user_data/paper/ + python scripts/reconcile.py --db user_data/tradesv3.docker.sqlite --log user_data/logs/bot.log + the Docker bot + +What the session ran is read from its files; --strategy, --pairs, --balance, +--start and --end override it. --no-download uses the data already on disk. + +Writes .reconcile.json next to the database; the panel's Research +tab shows the newest result per strategy. +""" + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from panel.common import USERDIR # noqa: E402 +from panel.replay import reconcile_path, replay_session # noqa: E402 +from ridgeline.reconcile import session_info # noqa: E402 + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("session", nargs="?", help="panel paper session id (user_data/paper/.sqlite)") + src.add_argument("--db", type=Path, help="any paper/dry-run Freqtrade SQLite database") + ap.add_argument("--log", type=Path, help="the bot's log file (default: .log)") + ap.add_argument("--strategy") + ap.add_argument("--pairs", nargs="+") + ap.add_argument("--balance", type=float) + ap.add_argument("--start", help="UTC, e.g. 2026-10-01 or '2026-10-01 14:00'") + ap.add_argument("--end", help="UTC (default: the bot's last log write)") + ap.add_argument("--no-download", action="store_true", help="don't top up candle data first") + args = ap.parse_args() + + db = args.db or USERDIR / "paper" / f"{args.session}.sqlite" + if not db.exists(): + sys.exit(f"No database at {db}") + info = session_info(db, log=args.log, strategy=args.strategy, pairs=args.pairs, balance=args.balance, + start=args.start, end=args.end) + print(f"{info.session_id}: {info.strategy}, {len(info.pairs)} pairs, {info.balance:g} starting balance, " + f"{info.start:%Y-%m-%d %H:%M} -> {info.end:%Y-%m-%d %H:%M} UTC") + print("Replaying the same period...") + result = replay_session(info, download=not args.no_download) + reconcile_path(db).write_text(json.dumps(result, indent=2, default=str)) + + p, r, m, g = result["paper"], result["replay"], result["matching"], result["gap"] + print(f"\n{'':10}{'trades':>8}{'closed':>8}{'realised':>11}{'return':>9}") + for name, s in (("paper", p), ("replay", r)): + print(f"{name:10}{s['trades']:>8}{s['closed']:>8}{s['realised_abs']:>11.2f}{s['return_pct']:>8.2f}%") + print(f"\nmatched {m['matched']}, paper only {m['paper_only']}, replay only {m['replay_only']}") + print(f"return gap {g['return_gap_pp']:+.2f} points = matched trades {g['matched_abs']:+.2f} " + f"+ paper-only {g['paper_only_abs']:+.2f} + replay-only {g['replay_only_abs']:+.2f} (stake currency)") + for row in result["trades"]: + print(f" {row['status']:<12}{row['pair']:<14}{row['paper_open'] or '-':<18}{row['replay_open'] or '-':<18}" + f"entry {_fmt(row['entry_cost_pct'])} exit {row['paper_exit'] or '-'} / {row['replay_exit'] or '-'}") + print() + for c in result["checks"]: + print(f" [{'PASS' if c['pass'] else 'FAIL'}] {c['label']} (got {c['value']})") + for w in result["warnings"]: + print(f" note: {w}") + verdict = "CLOSE to the replay" if result["close_to_replay"] else "NOT close to the replay (yet)" + print(f"\nPaper trading is {verdict}.\nwritten to {reconcile_path(db)}") + + +def _fmt(cost) -> str: + return "-" if cost is None else f"{cost:+.3f}%" + + +if __name__ == "__main__": + main() diff --git a/tests/test_panel_api.py b/tests/test_panel_api.py index 056d097..283c394 100644 --- a/tests/test_panel_api.py +++ b/tests/test_panel_api.py @@ -1,5 +1,6 @@ """Panel API: input validation, the dry-run safety guard, and the kill switch.""" +import json from types import SimpleNamespace import pytest @@ -66,3 +67,71 @@ def fake_call(method, path, **kw): def test_report_rejects_path_tricks(client): assert client.get("/api/report", params={"session": "../config"}).status_code == 400 + + +# ---------------------------------------------------------------- paper vs replay + +@pytest.fixture +def userdir(tmp_path, monkeypatch): + monkeypatch.setattr(app_mod, "USERDIR", tmp_path) + (tmp_path / "paper").mkdir(exist_ok=True) + return tmp_path + + +def make_session(userdir, session_id, meta=None): + import sqlite3 + + with sqlite3.connect(userdir / "paper" / f"{session_id}.sqlite") as con: + con.execute("CREATE TABLE trades (pair TEXT, strategy TEXT)") + con.close() + if meta: + (userdir / "paper" / f"{session_id}.meta.json").write_text(json.dumps(meta)) + + +def test_reconcile_rejects_bad_and_unknown_sessions(client, userdir): + assert client.post("/api/paper/sessions/bad.id/reconcile").status_code == 400 + assert client.get("/api/paper/sessions/20990101-000000/reconcile").status_code == 404 + + +def test_reconcile_needs_to_know_the_pairs(client, userdir): + make_session(userdir, "s1") + r = client.post("/api/paper/sessions/s1/reconcile") + assert r.status_code == 409 and "which" in r.json()["detail"] + + +def test_reconcile_submits_what_the_session_ran(client, userdir, monkeypatch): + make_session(userdir, "s2", {"strategy": "TrendRider4h", "pairs": ["BTC/USDT"], "balance": 300, + "started": "2026-09-01T00:00:00+00:00"}) + submitted = [] + monkeypatch.setattr(app_mod.session_replays, "submit", + lambda info: submitted.append(info) or {"status": "running", "session_id": info.session_id}) + assert client.post("/api/paper/sessions/s2/reconcile").json()["status"] == "running" + assert (submitted[0].strategy, submitted[0].pairs, submitted[0].balance) == ("TrendRider4h", ["BTC/USDT"], 300) + assert client.get("/api/paper/sessions/s2/reconcile").json() == {"status": "none"} + + +def test_research_shows_newest_reconciliation_per_strategy(client, userdir): + (userdir / "research").mkdir() + (userdir / "research" / "latest.json").write_text(json.dumps({"strategies": {}})) + + def result(session_id, generated, ok, status="done"): + return {"status": status, "generated": generated, "close_to_replay": ok, + "session": {"session_id": session_id, "strategy": "TrendRider4h"}, "period": {"days": 30.0}, + "checks": [{"pass": ok}, {"pass": True}]} + + for sid, gen, ok, status in [("old", "2026-10-01T00:00:00+00:00", False, "done"), + ("new", "2026-10-09T00:00:00+00:00", True, "done"), + ("broken", "2026-10-10T00:00:00+00:00", True, "error")]: + (userdir / "paper" / f"{sid}.reconcile.json").write_text(json.dumps(result(sid, gen, ok, status))) + checks = client.get("/api/research").json()["paper_checks"] + assert checks["TrendRider4h"]["session_id"] == "new" and checks["TrendRider4h"]["passed"] == 2 + + +def test_reconcile_refuses_until_the_first_candle_closes(client, userdir): + from datetime import datetime, timezone + + make_session(userdir, "s3", {"strategy": "TrendRider4h", "pairs": ["BTC/USDT"], "balance": 300, + "started": datetime.now(timezone.utc).isoformat()}) + r = client.post("/api/paper/sessions/s3/reconcile") + assert r.status_code == 409 and "Nothing to replay yet" in r.json()["detail"] + assert not (userdir / "paper" / "s3.reconcile.json").exists() diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py new file mode 100644 index 0000000..852fa7a --- /dev/null +++ b/tests/test_reconcile.py @@ -0,0 +1,199 @@ +"""Paper vs replay reconciliation: session discovery, trade matching, checks.""" + +import json +import sqlite3 + +import pandas as pd +import pytest + +from ridgeline.reconcile import ( + SessionInfo, + load_paper_trades, + load_replay_trades, + match_trades, + reconcile, + session_info, + strategy_fingerprint, + strategy_timeframe, +) + +T = lambda s: pd.Timestamp(s, tz="UTC") # noqa: E731 +H4 = pd.Timedelta("4h") + + +def trade(pair, opened, closed=None, open_rate=100.0, close_rate=None, profit_abs=None, exit_reason=None, stake=100.0): + is_open = closed is None + return {"pair": pair, "open_date": T(opened), "close_date": None if is_open else T(closed), + "open_rate": open_rate, "close_rate": close_rate, "stake": stake, + "profit_ratio": None if is_open else profit_abs / stake, "profit_abs": profit_abs, + "exit_reason": exit_reason, "is_open": is_open} + + +def frame(*trades): + df = pd.DataFrame(list(trades), columns=["pair", "open_date", "close_date", "open_rate", "close_rate", "stake", + "profit_ratio", "profit_abs", "exit_reason", "is_open"]) + for col in ("open_date", "close_date"): + df[col] = pd.to_datetime(df[col], utc=True) + return df + + +def info(start="2026-09-01 14:09", end="2026-10-05 10:00", balance=1000.0): + return SessionInfo(session_id="s1", db=None, strategy="TrendRider4h", pairs=["BTC/USDT", "ETH/USDT"], + balance=balance, start=T(start), end=T(end)) + + +def test_matching_is_by_pair_and_nearest_entry(): + paper = frame(trade("BTC/USDT", "2026-09-02 08:00:20"), trade("ETH/USDT", "2026-09-03 00:00:30"), + trade("BTC/USDT", "2026-09-10 12:01")) + replay = frame(trade("BTC/USDT", "2026-09-02 08:00"), trade("BTC/USDT", "2026-09-02 04:00"), + trade("SOL/USDT", "2026-09-03 00:00"), trade("BTC/USDT", "2026-09-10 00:00")) + matched, paper_only, replay_only = match_trades(paper, replay, H4 * 1.5) + assert matched == [(0, 0)] # the 08:00 entry, not the 04:00 one + assert paper_only == [1, 2] # ETH has no partner; 12h apart is too far + assert replay_only == [1, 2, 3] + + +def test_reconcile_breaks_down_the_gap(): + paper = frame( + # same signal, paper paid 0.2% more and got 0.1% less on the same exit + trade("BTC/USDT", "2026-09-02 08:00:20", "2026-09-05 12:00:05", 100.2, 109.89, 9.0, "exit_signal"), + trade("ETH/USDT", "2026-09-12 00:00:30", "2026-09-14 04:00:10", 50.0, 49.0, -2.0, "stop_loss"), + # opened after the replay's data ends: not compared yet + trade("BTC/USDT", "2026-10-05 09:00"), + ) + replay = frame( + trade("BTC/USDT", "2026-09-02 08:00", "2026-09-05 12:00", 100.0, 110.0, 10.0, "exit_signal"), + trade("SOL/USDT", "2026-09-20 00:00", "2026-09-22 00:00", 20.0, 21.0, 5.0, "exit_signal"), + trade("ETH/USDT", "2026-10-04 20:00"), # left open at the end of the replay + ) + events = [{"ts": "2026-09-15T10:00:00+00:00", "event": "entries_paused"}, + {"ts": "2026-09-15T12:00:00+00:00", "event": "trading_resumed"}, + {"ts": "2026-09-15T12:00:00+00:00", "event": "stake_capped"}, # modelled by the backtest + {"ts": "2026-08-01T00:00:00+00:00", "event": "flatten"}] # before the session + r = reconcile(paper, replay, info=info(), replay_end=T("2026-10-05 04:00"), timeframe="4h", events=events) + + assert r["period"]["start"].startswith("2026-09-01T12:00") # floored to the 4h candle + assert r["matching"] == {"matched": 1, "paper_only": 1, "replay_only": 2, "match_rate_pct": 25.0, "tolerance_hours": 6.0} + assert r["paper"]["trades"] == 2 and r["paper"]["realised_abs"] == 7.0 + assert r["replay"]["open"] == 1 and r["replay"]["realised_abs"] == 15.0 + g = r["gap"] + assert g["total_abs"] == -8.0 + assert g["matched_abs"] + g["paper_only_abs"] + g["replay_only_abs"] == pytest.approx(g["total_abs"]) + assert g["return_gap_pp"] == pytest.approx(-0.8) + assert r["fills"]["entry_cost_pct"] == pytest.approx(0.2) and r["fills"]["exit_cost_pct"] == pytest.approx(0.1) + assert r["fills"]["fill_cost_pct"] == pytest.approx(0.15) + + checks = {c["key"]: c for c in r["checks"]} + assert checks["days"]["pass"] and checks["return_gap_pp"]["pass"] and checks["fill_cost_pct"]["pass"] + assert not checks["match_rate_pct"]["pass"] and r["close_to_replay"] is False + assert any("1 paper trade(s) opened after" in w for w in r["warnings"]) + assert any("entries_paused x1, trading_resumed x1" in w for w in r["warnings"]) + assert not any("flatten" in w or "stake_capped" in w for w in r["warnings"]) + assert [row["status"] for row in r["trades"]] == ["matched", "paper_only", "replay_only", "replay_only"] + assert r["trades"][-1]["replay_exit"] == "open" + + +def test_a_faithful_month_passes(): + days = range(2, 28, 3) + paper = frame(*(trade("BTC/USDT", f"2026-09-{d:02d} 04:00:30", f"2026-09-{d + 1:02d} 08:00:10", + 100.05, 101.0, 1.0, "exit_signal") for d in days)) + replay = frame(*(trade("BTC/USDT", f"2026-09-{d:02d} 04:00", f"2026-09-{d + 1:02d} 08:00", + 100.0, 101.0, 1.05, "exit_signal") for d in days)) + r = reconcile(paper, replay, info=info(start="2026-09-01 00:05"), replay_end=T("2026-10-05 04:00"), timeframe="4h") + assert r["matching"]["match_rate_pct"] == 100.0 + assert r["fills"]["same_exit_reason_pct"] == 100.0 + assert r["close_to_replay"] is True, r["checks"] + + +def test_short_session_without_trades_fails_honestly(): + r = reconcile(frame(), frame(), info=info(start="2026-10-01 00:00"), replay_end=T("2026-10-05 04:00"), timeframe="4h") + assert r["close_to_replay"] is False + assert {c["key"]: c["value"] for c in r["checks"]}["match_rate_pct"] is None + assert any("nothing to compare" in w for w in r["warnings"]) + + +def test_stale_replay_data_is_flagged(): + r = reconcile(frame(), frame(), info=info(), replay_end=T("2026-09-20 00:00"), timeframe="4h") + assert any("Download fresh data" in w for w in r["warnings"]) + + +# ---------------------------------------------------------------- session files + +def make_db(path, start="2026-09-01 14:09:52.1", strategy="TrendRider4h"): + with sqlite3.connect(path) as con: + con.execute("CREATE TABLE KeyValueStore (id INTEGER, key TEXT, value_type TEXT, string_value TEXT, " + "datetime_value DATETIME, float_value FLOAT, int_value INTEGER)") + con.execute("INSERT INTO KeyValueStore VALUES (1, 'bot_start_time', 'datetime', NULL, ?, NULL, NULL)", (start,)) + con.execute("CREATE TABLE trades (pair TEXT, open_date DATETIME, close_date DATETIME, open_rate FLOAT, " + "close_rate FLOAT, stake_amount FLOAT, close_profit FLOAT, close_profit_abs FLOAT, " + "exit_reason TEXT, is_open BOOLEAN, strategy TEXT)") + con.execute("INSERT INTO trades VALUES ('BTC/USDT', '2026-09-02 08:00:20.5', '2026-09-05 12:00:05', " + "100.2, 109.89, 100, 0.09, 9.0, 'exit_signal', 0, ?)", (strategy,)) + con.execute("INSERT INTO trades VALUES ('ETH/USDT', '2026-09-12 00:00:30', NULL, 50, NULL, 100, NULL, NULL, " + "NULL, 1, ?)", (strategy,)) + con.close() + + +def test_session_info_from_log_and_database(tmp_path): + db = tmp_path / "20260901-010946.sqlite" + make_db(db) + (tmp_path / "20260901-010946.config.json").write_text(json.dumps({"dry_run_wallet": 500})) + (tmp_path / "20260901-010946.log.1").write_text( + "... - INFO - Using resolved strategy TrendRider4h from 'x'...\n" + "... - INFO - Whitelist with 2 pairs: ['BTC/USDT', 'ETH/USDT']\n") + (tmp_path / "20260901-010946.log").write_text("... - INFO - Whitelist with 1 pairs: ['BTC/USDT']\n") + s = session_info(db) + assert (s.strategy, s.pairs, s.balance) == ("TrendRider4h", ["BTC/USDT", "ETH/USDT"], 500) # oldest log first + assert s.start == T("2026-09-01 14:09:52.1") + assert abs(s.end - pd.Timestamp.now(tz="UTC")) < pd.Timedelta(minutes=5) # last log write + assert any("pair list changed" in w for w in s.warnings) + + trades = load_paper_trades(db) + assert list(trades["is_open"]) == [False, True] and trades["open_date"].dt.tz is not None + + +def test_meta_file_wins_and_code_changes_are_flagged(tmp_path): + db = tmp_path / "s.sqlite" + make_db(db) + code = strategy_fingerprint("TrendRider4h") + code["TrendRider4h.py"] = "0" * 16 + (tmp_path / "s.meta.json").write_text(json.dumps({ + "strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 250, "max_open_trades": 2, + "started": "2026-09-03T00:00:00+00:00", "code": code})) + s = session_info(db, end="2026-10-01") + assert (s.pairs, s.balance, s.max_open_trades, s.start) == (["SOL/USDT"], 250, 2, T("2026-09-03")) + assert s.end == T("2026-10-01") + assert any("TrendRider4h.py" in w for w in s.warnings) + assert not any("TrendRider4h.json" in w for w in s.warnings) + + +def test_missing_pairs_need_to_be_given(tmp_path): + db = tmp_path / "s.sqlite" + make_db(db) + with pytest.raises(ValueError, match="which pairs"): + session_info(db) + assert session_info(db, pairs=["BTC/USDT"]).pairs == ["BTC/USDT"] + + +def test_strategy_timeframes(): + assert strategy_timeframe("TrendRider4h") == "4h" + assert strategy_timeframe("EmaRsiMomentum") == "1h" + + +def test_replay_trades_left_open_come_back_open(tmp_path): + def bt_trade(pair, opened, closed, reason): + return {"pair": pair, "open_date": opened, "close_date": closed, "open_rate": 1.0, "close_rate": 1.1, + "stake_amount": 100.0, "profit_ratio": 0.1, "profit_abs": 10.0, "exit_reason": reason, "is_open": False} + + result = {"strategy": {"TrendRider4h": { + "backtest_start": "2026-09-01 12:00:00", "backtest_end": "2026-10-05 04:00:00", "timeframe": "4h", + "starting_balance": 1000, "trades": [ + bt_trade("BTC/USDT", "2026-09-02 08:00:00+00:00", "2026-09-05 12:00:00+00:00", "force_exit"), # flatten-like, mid-run + bt_trade("ETH/USDT", "2026-10-04 20:00:00+00:00", "2026-10-05 04:00:00+00:00", "force_exit"), + ]}}} + (tmp_path / "backtest-result-x.json").write_text(json.dumps(result)) + (tmp_path / ".last_result.json").write_text(json.dumps({"latest_backtest": "backtest-result-x.json"})) + df, meta = load_replay_trades(tmp_path) + assert list(df["is_open"]) == [False, True] + assert pd.isna(df.loc[1, "profit_abs"]) and df.loc[0, "profit_abs"] == 10.0 + assert meta["end"] == T("2026-10-05 04:00") and meta["timeframe"] == "4h"