From e622003d3c0b1f3dfebc7dbca3b68ed7caa20c66 Mon Sep 17 00:00:00 2001 From: devam <56464719+devam2001@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:24:09 +1100 Subject: [PATCH] Resume paper sessions after a restart; flag bot downtime A 4-week paper test was lost to any reboot, crash or sleep: the panel could only start a new session. - Resume restarts the last session's bot on the same database, so the wallet, open trades and history carry on. Sessions pin their pair list in their own config, so the monthly config.json update can't change a running test; older sessions get theirs back from the bot log. - The paper-vs-replay check finds periods with no log output for 10+ minutes and reports the downtime, since the replay trades straight through them. - Tests: resume, pinned pairs, the dry-run guard on resume, the risk-control resume staying separate, and gap detection. --- README.md | 2 ++ docs/OPERATIONS.md | 4 +++ panel/app.py | 8 +++++ panel/bot.py | 80 +++++++++++++++++++++++++++++++---------- panel/static/app.js | 2 ++ panel/static/index.html | 1 + ridgeline/reconcile.py | 33 +++++++++++++++-- tests/test_panel_api.py | 56 +++++++++++++++++++++++++++++ tests/test_reconcile.py | 12 +++++++ 9 files changed, 178 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 2251ea1..ed58724 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ Open **http://127.0.0.1:8090**. strategy, press **Start**. The bot trades live Binance prices without sending orders. **Reset wallet** starts a fresh session with a new balance; every session keeps its own database in `user_data/paper/`, so old results aren't lost. + **Resume** restarts the last session after a reboot, crash or Stop, with the + same wallet, open trades and pair list. A 4-week paper test survives restarts. While it runs, **Open FreqUI** (http://127.0.0.1:8080, log in with the username/password from `config.private.json`) shows charts, per-trade detail and more. - **Replay:** pick strategy, balance and date range. It simulates that period in diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0d01ed7..74becad 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -231,6 +231,10 @@ The result notes things the replay can't reproduce, so check those first: - drawdown halts, which need a manual resume live but resume automatically in backtests - strategy code or parameters changed during the session - paper trades newer than the latest closed candle +- time the bot was offline (a reboot, sleep or crash): any stretch of 10+ + minutes without log output. The replay keeps trading through these gaps, so + keep downtime short. After a restart, press **Resume** rather than Reset: + Resume continues the same session. **If it fails.** Don't loosen the targets. Find out why the two diverged. If the backtest assumptions are wrong (for example, real fills are consistently diff --git a/panel/app.py b/panel/app.py index 709a99f..2904fff 100644 --- a/panel/app.py +++ b/panel/app.py @@ -100,6 +100,14 @@ def paper_reset(req: PaperReset): raise HTTPException(409, str(e)) +@app.post("/api/paper/resume") +def paper_resume(): + try: + return bot.resume_session() + except PaperBotError as e: + raise HTTPException(409, str(e)) + + @app.get("/api/mode") def mode(): return {"monitor": MONITOR_MODE, "version": __version__} diff --git a/panel/bot.py b/panel/bot.py index 5f232ca..b78a5e1 100644 --- a/panel/bot.py +++ b/panel/bot.py @@ -29,7 +29,7 @@ from panel.common import FREQTRADE, ROOT, USERDIR, config_args, load_json, merged_config from ridgeline import audit -from ridgeline.reconcile import strategy_fingerprint +from ridgeline.reconcile import session_info, strategy_fingerprint from ridgeline.risk import RiskLimits, build_state, limits_summary, load_halt, set_halt MONITOR_MODE = os.environ.get("PANEL_MODE", "").lower() == "monitor" @@ -87,11 +87,9 @@ def _our_process(self) -> Optional[psutil.Process]: # --- actions --- - def start(self, balance: float, strategy: str) -> dict: + def _preflight(self) -> None: if MONITOR_MODE: raise PaperBotError("Process control is disabled in monitor mode (Docker manages the bot).") - if balance <= 0: - raise PaperBotError("Starting balance must be positive.") if self._our_process(): raise PaperBotError("A paper session is already running. Stop or reset it first.") if _port_open(self.base_url): @@ -100,30 +98,84 @@ def start(self, balance: float, strategy: str) -> dict: "started by hand). Stop it first - the panel will not kill processes it didn't start." ) + def start(self, balance: float, strategy: str) -> dict: + if balance <= 0: + raise PaperBotError("Starting balance must be positive.") + self._preflight() + session_id = datetime.now().strftime("%Y%m%d-%H%M%S") + cfg = merged_config() + pairs = cfg.get("exchange", {}).get("pair_whitelist", []) session_cfg = PAPER_DIR / f"{session_id}.config.json" session_cfg.write_text(json.dumps({ "dry_run": True, "dry_run_wallet": balance, "db_url": f"sqlite:///user_data/paper/{session_id}.sqlite", "bot_name": f"paper-{session_id}", - "exchange": {"key": "", "secret": ""}, + # Pin the pair list: config.json's changes monthly, and a session + # resumed after a restart must keep trading what it started with. + "exchange": {"key": "", "secret": "", "pair_whitelist": pairs}, }, indent=2)) args = [*config_args(), "-c", str(session_cfg)] self._verify_dry_run(args) # What this session runs, for reconciling it with a replay later: # config.json's pair list changes monthly, and code can change mid-session. - cfg = merged_config() (PAPER_DIR / f"{session_id}.meta.json").write_text(json.dumps({ "session_id": session_id, "strategy": strategy, "balance": balance, - "pairs": cfg.get("exchange", {}).get("pair_whitelist", []), + "pairs": pairs, "max_open_trades": cfg.get("max_open_trades"), "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), "code": strategy_fingerprint(strategy), }, indent=2)) + state = { + "pid": self._spawn(session_id, args, strategy), + "session_id": session_id, + "strategy": strategy, + "balance": balance, + "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + self._save_state(state) + audit.append("paper_session_started", "panel", + {"session_id": session_id, "strategy": strategy, "balance": balance}) + return state + + def resume_session(self) -> dict: + """Restart the last session's bot on the same database, after a reboot, + a crash or a Stop. Open trades, the wallet and the trade history carry + on, so a weeks-long paper test isn't lost to a restart.""" + self._preflight() + state = self._state() + session_id = state.get("session_id") + if not session_id: + raise PaperBotError("There's no previous paper session to resume.") + session_cfg = PAPER_DIR / f"{session_id}.config.json" + db = PAPER_DIR / f"{session_id}.sqlite" + if not session_cfg.exists() or not db.exists(): + raise PaperBotError(f"Session {session_id}'s files are missing. Start a new session instead.") + try: + info = session_info(db) + except ValueError as e: + raise PaperBotError(f"Can't resume session {session_id}: {e}") from e + cfg = load_json(session_cfg, {}) or {} + if not cfg.get("exchange", {}).get("pair_whitelist"): + # Sessions from before pairs were pinned: keep what they traded, + # not whatever config.json lists today. + cfg.setdefault("exchange", {}).update(key="", secret="", pair_whitelist=info.pairs) + session_cfg.write_text(json.dumps(cfg, indent=2)) + args = [*config_args(), "-c", str(session_cfg)] + self._verify_dry_run(args) + strategy = state.get("strategy") or info.strategy + state.update(pid=self._spawn(session_id, args, strategy), strategy=strategy, + resumed=[*state.get("resumed", []), datetime.now(timezone.utc).isoformat(timespec="seconds")]) + state.pop("stopped", None) + self._save_state(state) + audit.append("paper_session_resumed", "panel", {"session_id": session_id, "strategy": strategy}) + return state + + def _spawn(self, session_id: str, args: list[str], strategy: str) -> int: log_path = PAPER_DIR / f"{session_id}.log" cmd = [*FREQTRADE, "trade", *args, "--strategy", strategy, "--logfile", str(log_path)] popen_kw = dict(cwd=ROOT, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, @@ -138,17 +190,7 @@ def start(self, balance: float, strategy: str) -> dict: proc = subprocess.Popen(cmd, creationflags=flags, **popen_kw) else: proc = subprocess.Popen(cmd, start_new_session=True, **popen_kw) - state = { - "pid": proc.pid, - "session_id": session_id, - "strategy": strategy, - "balance": balance, - "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), - } - self._save_state(state) - audit.append("paper_session_started", "panel", - {"session_id": session_id, "strategy": strategy, "balance": balance}) - return state + return proc.pid def _verify_dry_run(self, args: list[str]) -> None: out = subprocess.run([*FREQTRADE, "show-config", *args], cwd=ROOT, @@ -289,6 +331,8 @@ def status(self) -> dict: "api_up": False, "session": state if ours else None, "last_session": state if not ours and state else None, + "can_resume": (not ours and not listening and not MONITOR_MODE and bool(state.get("session_id")) + and (PAPER_DIR / f"{state.get('session_id')}.sqlite").exists()), "freqtrade_ui": self.base_url, } if not listening: diff --git a/panel/static/app.js b/panel/static/app.js index 6a673c9..9536f50 100644 --- a/panel/static/app.js +++ b/panel/static/app.js @@ -130,6 +130,7 @@ async function refreshPaper() { $("#paper-start").disabled = paperBusy || st.managed || st.external || monitorMode; $("#paper-stop").disabled = paperBusy || !st.managed || monitorMode; + $("#paper-resume").disabled = paperBusy || !st.can_resume; $("#paper-reset").disabled = paperBusy || st.external || monitorMode; ["#risk-pause", "#risk-resume", "#risk-flatten"].forEach((id) => ($(id).disabled = paperBusy || !st.api_up)); if (st.api_up) refreshRisk(); else $("#risk-meters").innerHTML = `

Available while a bot is running.

`; @@ -196,6 +197,7 @@ $("#paper-form").addEventListener("submit", (e) => { paperAction(() => api("/api/paper/start", { method: "POST", body: { balance: +$("#paper-balance").value, strategy: $("#paper-strategy").value } })); }); $("#paper-stop").addEventListener("click", () => paperAction(() => api("/api/paper/stop", { method: "POST" }))); +$("#paper-resume").addEventListener("click", () => paperAction(() => api("/api/paper/resume", { method: "POST" }))); $("#paper-reset").addEventListener("click", () => { const bal = +$("#paper-balance").value; if (!confirm(`Start a fresh paper session with ${bal} USDT? The current session's trades are kept in its own archive file.`)) return; diff --git a/panel/static/index.html b/panel/static/index.html index a99af99..76519b3 100644 --- a/panel/static/index.html +++ b/panel/static/index.html @@ -55,6 +55,7 @@

Fake wallet

+
diff --git a/ridgeline/reconcile.py b/ridgeline/reconcile.py index 30dcf22..9e68b9a 100644 --- a/ridgeline/reconcile.py +++ b/ridgeline/reconcile.py @@ -75,11 +75,12 @@ class SessionInfo: end: pd.Timestamp max_open_trades: Optional[int] = None warnings: list[str] = field(default_factory=list) + downtime_hours: float = 0.0 def to_dict(self) -> dict: return {"session_id": self.session_id, "strategy": self.strategy, "pairs": self.pairs, "balance": self.balance, "start": self.start.isoformat(), "end": self.end.isoformat(), - "max_open_trades": self.max_open_trades} + "max_open_trades": self.max_open_trades, "downtime_hours": self.downtime_hours} def strategy_fingerprint(strategy: str, strategy_dir: Path = STRATEGY_DIR) -> dict[str, Optional[str]]: @@ -154,8 +155,36 @@ def session_info(db: Path, *, log: Optional[Path] = None, strategy: Optional[str warnings.append(f"Changed since the session started: {', '.join(changed)}. " "The replay runs the current version, so some differences are expected.") + gaps = log_gaps(logs) + downtime = sum((b - a for a, b in gaps), pd.Timedelta(0)) + if gaps: + longest = max(b - a for a, b in gaps) + warnings.append(f"The bot was offline {len(gaps)} time(s), {_hours(downtime)} in total (longest " + f"{_hours(longest)}, from {gaps[0][0]:%Y-%m-%d %H:%M} local time on). The replay trades through " + "those gaps, so trades it takes while the bot was down are missing from paper.") + return SessionInfo(session_id=db.stem, db=db, strategy=strategy, pairs=list(pairs), balance=float(balance), - start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings) + start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings, + downtime_hours=round(downtime.total_seconds() / 3600, 2)) + + +# A running bot logs a heartbeat every minute; silence longer than this means it was down. +OFFLINE_AFTER = pd.Timedelta(minutes=10) +_LOG_TIME_RE = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})", re.M) + + +def log_gaps(logs: list[Path], threshold: pd.Timedelta = OFFLINE_AFTER) -> list[tuple[pd.Timestamp, pd.Timestamp]]: + """Periods with no log output for longer than `threshold`, in the bot's + local time: a reboot, a crash, the PC asleep, or a Stop and a Resume.""" + text = [m for p in logs for m in _LOG_TIME_RE.findall(p.read_text(encoding="utf-8", errors="replace"))] + stamps = pd.Series(pd.to_datetime(text, format="%Y-%m-%d %H:%M:%S")).sort_values().reset_index(drop=True) + gap = stamps.diff() > threshold + return [(stamps[i - 1], stamps[i]) for i in gap[gap].index] + + +def _hours(delta: pd.Timedelta) -> str: + hours = delta.total_seconds() / 3600 + return f"{hours:.1f} h" if hours < 48 else f"{hours / 24:.1f} days" def _load_json(path: Path) -> dict: diff --git a/tests/test_panel_api.py b/tests/test_panel_api.py index 283c394..acde5ce 100644 --- a/tests/test_panel_api.py +++ b/tests/test_panel_api.py @@ -135,3 +135,59 @@ def test_reconcile_refuses_until_the_first_candle_closes(client, userdir): r = client.post("/api/paper/sessions/s3/reconcile") assert r.status_code == 409 and "Nothing to replay yet" in r.json()["detail"] assert not (userdir / "paper" / "s3.reconcile.json").exists() + + +def test_resume_needs_a_previous_session(client): + r = client.post("/api/paper/resume") + assert r.status_code == 409 and "no previous" in r.json()["detail"] + + +def test_resume_restarts_the_same_session_with_its_own_pairs(client, monkeypatch): + paper = bot_mod.PAPER_DIR + make_session(paper.parent, "s9", {"strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 100, + "started": "2026-10-01T00:00:00+00:00"}) + (paper / "s9.config.json").write_text(json.dumps({"dry_run": True, "dry_run_wallet": 100, + "exchange": {"key": "", "secret": ""}})) + (paper / "state.json").write_text(json.dumps({"pid": None, "session_id": "s9", "strategy": "TrendRider4h", + "balance": 100, "stopped": "2026-10-05T00:00:00+00:00"})) + launched = [] + monkeypatch.setattr(bot_mod, "_port_open", lambda url: False) + monkeypatch.setattr(bot_mod.subprocess, "run", lambda *a, **k: SimpleNamespace(stdout='{"dry_run": true}', stderr="")) + monkeypatch.setattr(bot_mod.subprocess, "Popen", lambda cmd, **k: launched.append(cmd) or SimpleNamespace(pid=4242)) + + assert client.get("/api/paper/status").json()["can_resume"] is True + state = client.post("/api/paper/resume").json() + assert state["session_id"] == "s9" and state["pid"] == 4242 and len(state["resumed"]) == 1 and "stopped" not in state + # pinned to what the session started with, not today's config.json list + assert json.loads((paper / "s9.config.json").read_text())["exchange"]["pair_whitelist"] == ["SOL/USDT"] + cmd = launched[0] + assert str(paper / "s9.config.json") in cmd and cmd[cmd.index("--strategy") + 1] == "TrendRider4h" + assert client.get("/api/audit").json()["events"][0]["event"] == "paper_session_resumed" + + +def test_resume_refuses_a_live_config(client, monkeypatch): + paper = bot_mod.PAPER_DIR + make_session(paper.parent, "s8", {"strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 100, + "started": "2026-10-01T00:00:00+00:00"}) + (paper / "s8.config.json").write_text(json.dumps({"dry_run": True})) + (paper / "state.json").write_text(json.dumps({"session_id": "s8", "strategy": "TrendRider4h"})) + monkeypatch.setattr(bot_mod, "_port_open", lambda url: False) + monkeypatch.setattr(bot_mod.subprocess, "run", lambda *a, **k: SimpleNamespace(stdout='{"dry_run": false}', stderr="")) + + def no_launch(*a, **k): + raise AssertionError("a bot process was launched") + + monkeypatch.setattr(bot_mod.subprocess, "Popen", no_launch) + r = client.post("/api/paper/resume") + assert r.status_code == 409 and "Refusing" in r.json()["detail"] + + +def test_risk_resume_restarts_trading_not_the_session(client, monkeypatch): + """/api/bot/resume is the risk control (restart entries, clear a halt); + /api/paper/resume restarts a stopped session. They must not be mixed up.""" + calls = [] + monkeypatch.setattr(app_mod.bot, "_call", lambda method, path, **kw: calls.append(path) or {"status": "ok"}) + monkeypatch.setattr(app_mod.bot, "resume_session", lambda: (_ for _ in ()).throw(AssertionError("session resumed"))) + monkeypatch.setattr(bot_mod, "set_halt", lambda *a, **k: None) + assert client.post("/api/bot/resume").status_code == 200 + assert calls == ["start"] diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py index 852fa7a..1ce0c86 100644 --- a/tests/test_reconcile.py +++ b/tests/test_reconcile.py @@ -197,3 +197,15 @@ def bt_trade(pair, opened, closed, reason): assert list(df["is_open"]) == [False, True] assert pd.isna(df.loc[1, "profit_abs"]) and df.loc[0, "profit_abs"] == 10.0 assert meta["end"] == T("2026-10-05 04:00") and meta["timeframe"] == "4h" + + +def test_offline_periods_are_found_in_the_log(tmp_path): + db = tmp_path / "s.sqlite" + make_db(db) + beats = pd.date_range("2026-09-02 10:00", periods=30, freq="1min").append( + pd.date_range("2026-09-02 13:30", periods=30, freq="1min")) # down from 10:29 to 13:30 + (tmp_path / "s.log").write_text("".join(f"{t:%Y-%m-%d %H:%M:%S},123 - freqtrade.worker - INFO - Bot heartbeat.\n" + for t in beats)) + s = session_info(db, pairs=["BTC/USDT"], end="2026-10-01") + assert s.downtime_hours == pytest.approx(3.02, abs=0.01) + assert any("offline 1 time(s)" in w and "3.0 h" in w for w in s.warnings)