diff --git a/README.md b/README.md index 2251ea1..ed58724 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ Open **http://127.0.0.1:8090**. strategy, press **Start**. The bot trades live Binance prices without sending orders. **Reset wallet** starts a fresh session with a new balance; every session keeps its own database in `user_data/paper/`, so old results aren't lost. + **Resume** restarts the last session after a reboot, crash or Stop, with the + same wallet, open trades and pair list. A 4-week paper test survives restarts. While it runs, **Open FreqUI** (http://127.0.0.1:8080, log in with the username/password from `config.private.json`) shows charts, per-trade detail and more. - **Replay:** pick strategy, balance and date range. It simulates that period in diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 0d01ed7..74becad 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -231,6 +231,10 @@ The result notes things the replay can't reproduce, so check those first: - drawdown halts, which need a manual resume live but resume automatically in backtests - strategy code or parameters changed during the session - paper trades newer than the latest closed candle +- time the bot was offline (a reboot, sleep or crash): any stretch of 10+ + minutes without log output. The replay keeps trading through these gaps, so + keep downtime short. After a restart, press **Resume** rather than Reset: + Resume continues the same session. **If it fails.** Don't loosen the targets. Find out why the two diverged. If the backtest assumptions are wrong (for example, real fills are consistently diff --git a/panel/app.py b/panel/app.py index 709a99f..2904fff 100644 --- a/panel/app.py +++ b/panel/app.py @@ -100,6 +100,14 @@ def paper_reset(req: PaperReset): raise HTTPException(409, str(e)) +@app.post("/api/paper/resume") +def paper_resume(): + try: + return bot.resume_session() + except PaperBotError as e: + raise HTTPException(409, str(e)) + + @app.get("/api/mode") def mode(): return {"monitor": MONITOR_MODE, "version": __version__} diff --git a/panel/bot.py b/panel/bot.py index 5f232ca..b78a5e1 100644 --- a/panel/bot.py +++ b/panel/bot.py @@ -29,7 +29,7 @@ from panel.common import FREQTRADE, ROOT, USERDIR, config_args, load_json, merged_config from ridgeline import audit -from ridgeline.reconcile import strategy_fingerprint +from ridgeline.reconcile import session_info, strategy_fingerprint from ridgeline.risk import RiskLimits, build_state, limits_summary, load_halt, set_halt MONITOR_MODE = os.environ.get("PANEL_MODE", "").lower() == "monitor" @@ -87,11 +87,9 @@ def _our_process(self) -> Optional[psutil.Process]: # --- actions --- - def start(self, balance: float, strategy: str) -> dict: + def _preflight(self) -> None: if MONITOR_MODE: raise PaperBotError("Process control is disabled in monitor mode (Docker manages the bot).") - if balance <= 0: - raise PaperBotError("Starting balance must be positive.") if self._our_process(): raise PaperBotError("A paper session is already running. Stop or reset it first.") if _port_open(self.base_url): @@ -100,30 +98,84 @@ def start(self, balance: float, strategy: str) -> dict: "started by hand). Stop it first - the panel will not kill processes it didn't start." ) + def start(self, balance: float, strategy: str) -> dict: + if balance <= 0: + raise PaperBotError("Starting balance must be positive.") + self._preflight() + session_id = datetime.now().strftime("%Y%m%d-%H%M%S") + cfg = merged_config() + pairs = cfg.get("exchange", {}).get("pair_whitelist", []) session_cfg = PAPER_DIR / f"{session_id}.config.json" session_cfg.write_text(json.dumps({ "dry_run": True, "dry_run_wallet": balance, "db_url": f"sqlite:///user_data/paper/{session_id}.sqlite", "bot_name": f"paper-{session_id}", - "exchange": {"key": "", "secret": ""}, + # Pin the pair list: config.json's changes monthly, and a session + # resumed after a restart must keep trading what it started with. + "exchange": {"key": "", "secret": "", "pair_whitelist": pairs}, }, indent=2)) args = [*config_args(), "-c", str(session_cfg)] self._verify_dry_run(args) # What this session runs, for reconciling it with a replay later: # config.json's pair list changes monthly, and code can change mid-session. - cfg = merged_config() (PAPER_DIR / f"{session_id}.meta.json").write_text(json.dumps({ "session_id": session_id, "strategy": strategy, "balance": balance, - "pairs": cfg.get("exchange", {}).get("pair_whitelist", []), + "pairs": pairs, "max_open_trades": cfg.get("max_open_trades"), "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), "code": strategy_fingerprint(strategy), }, indent=2)) + state = { + "pid": self._spawn(session_id, args, strategy), + "session_id": session_id, + "strategy": strategy, + "balance": balance, + "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + self._save_state(state) + audit.append("paper_session_started", "panel", + {"session_id": session_id, "strategy": strategy, "balance": balance}) + return state + + def resume_session(self) -> dict: + """Restart the last session's bot on the same database, after a reboot, + a crash or a Stop. Open trades, the wallet and the trade history carry + on, so a weeks-long paper test isn't lost to a restart.""" + self._preflight() + state = self._state() + session_id = state.get("session_id") + if not session_id: + raise PaperBotError("There's no previous paper session to resume.") + session_cfg = PAPER_DIR / f"{session_id}.config.json" + db = PAPER_DIR / f"{session_id}.sqlite" + if not session_cfg.exists() or not db.exists(): + raise PaperBotError(f"Session {session_id}'s files are missing. Start a new session instead.") + try: + info = session_info(db) + except ValueError as e: + raise PaperBotError(f"Can't resume session {session_id}: {e}") from e + cfg = load_json(session_cfg, {}) or {} + if not cfg.get("exchange", {}).get("pair_whitelist"): + # Sessions from before pairs were pinned: keep what they traded, + # not whatever config.json lists today. + cfg.setdefault("exchange", {}).update(key="", secret="", pair_whitelist=info.pairs) + session_cfg.write_text(json.dumps(cfg, indent=2)) + args = [*config_args(), "-c", str(session_cfg)] + self._verify_dry_run(args) + strategy = state.get("strategy") or info.strategy + state.update(pid=self._spawn(session_id, args, strategy), strategy=strategy, + resumed=[*state.get("resumed", []), datetime.now(timezone.utc).isoformat(timespec="seconds")]) + state.pop("stopped", None) + self._save_state(state) + audit.append("paper_session_resumed", "panel", {"session_id": session_id, "strategy": strategy}) + return state + + def _spawn(self, session_id: str, args: list[str], strategy: str) -> int: log_path = PAPER_DIR / f"{session_id}.log" cmd = [*FREQTRADE, "trade", *args, "--strategy", strategy, "--logfile", str(log_path)] popen_kw = dict(cwd=ROOT, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, @@ -138,17 +190,7 @@ def start(self, balance: float, strategy: str) -> dict: proc = subprocess.Popen(cmd, creationflags=flags, **popen_kw) else: proc = subprocess.Popen(cmd, start_new_session=True, **popen_kw) - state = { - "pid": proc.pid, - "session_id": session_id, - "strategy": strategy, - "balance": balance, - "started": datetime.now(timezone.utc).isoformat(timespec="seconds"), - } - self._save_state(state) - audit.append("paper_session_started", "panel", - {"session_id": session_id, "strategy": strategy, "balance": balance}) - return state + return proc.pid def _verify_dry_run(self, args: list[str]) -> None: out = subprocess.run([*FREQTRADE, "show-config", *args], cwd=ROOT, @@ -289,6 +331,8 @@ def status(self) -> dict: "api_up": False, "session": state if ours else None, "last_session": state if not ours and state else None, + "can_resume": (not ours and not listening and not MONITOR_MODE and bool(state.get("session_id")) + and (PAPER_DIR / f"{state.get('session_id')}.sqlite").exists()), "freqtrade_ui": self.base_url, } if not listening: diff --git a/panel/static/app.js b/panel/static/app.js index 6a673c9..9536f50 100644 --- a/panel/static/app.js +++ b/panel/static/app.js @@ -130,6 +130,7 @@ async function refreshPaper() { $("#paper-start").disabled = paperBusy || st.managed || st.external || monitorMode; $("#paper-stop").disabled = paperBusy || !st.managed || monitorMode; + $("#paper-resume").disabled = paperBusy || !st.can_resume; $("#paper-reset").disabled = paperBusy || st.external || monitorMode; ["#risk-pause", "#risk-resume", "#risk-flatten"].forEach((id) => ($(id).disabled = paperBusy || !st.api_up)); if (st.api_up) refreshRisk(); else $("#risk-meters").innerHTML = `

Available while a bot is running.

`; @@ -196,6 +197,7 @@ $("#paper-form").addEventListener("submit", (e) => { paperAction(() => api("/api/paper/start", { method: "POST", body: { balance: +$("#paper-balance").value, strategy: $("#paper-strategy").value } })); }); $("#paper-stop").addEventListener("click", () => paperAction(() => api("/api/paper/stop", { method: "POST" }))); +$("#paper-resume").addEventListener("click", () => paperAction(() => api("/api/paper/resume", { method: "POST" }))); $("#paper-reset").addEventListener("click", () => { const bal = +$("#paper-balance").value; if (!confirm(`Start a fresh paper session with ${bal} USDT? The current session's trades are kept in its own archive file.`)) return; diff --git a/panel/static/index.html b/panel/static/index.html index a99af99..76519b3 100644 --- a/panel/static/index.html +++ b/panel/static/index.html @@ -55,6 +55,7 @@

Fake wallet

+
diff --git a/ridgeline/reconcile.py b/ridgeline/reconcile.py index 30dcf22..9e68b9a 100644 --- a/ridgeline/reconcile.py +++ b/ridgeline/reconcile.py @@ -75,11 +75,12 @@ class SessionInfo: end: pd.Timestamp max_open_trades: Optional[int] = None warnings: list[str] = field(default_factory=list) + downtime_hours: float = 0.0 def to_dict(self) -> dict: return {"session_id": self.session_id, "strategy": self.strategy, "pairs": self.pairs, "balance": self.balance, "start": self.start.isoformat(), "end": self.end.isoformat(), - "max_open_trades": self.max_open_trades} + "max_open_trades": self.max_open_trades, "downtime_hours": self.downtime_hours} def strategy_fingerprint(strategy: str, strategy_dir: Path = STRATEGY_DIR) -> dict[str, Optional[str]]: @@ -154,8 +155,36 @@ def session_info(db: Path, *, log: Optional[Path] = None, strategy: Optional[str warnings.append(f"Changed since the session started: {', '.join(changed)}. " "The replay runs the current version, so some differences are expected.") + gaps = log_gaps(logs) + downtime = sum((b - a for a, b in gaps), pd.Timedelta(0)) + if gaps: + longest = max(b - a for a, b in gaps) + warnings.append(f"The bot was offline {len(gaps)} time(s), {_hours(downtime)} in total (longest " + f"{_hours(longest)}, from {gaps[0][0]:%Y-%m-%d %H:%M} local time on). The replay trades through " + "those gaps, so trades it takes while the bot was down are missing from paper.") + return SessionInfo(session_id=db.stem, db=db, strategy=strategy, pairs=list(pairs), balance=float(balance), - start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings) + start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings, + downtime_hours=round(downtime.total_seconds() / 3600, 2)) + + +# A running bot logs a heartbeat every minute; silence longer than this means it was down. +OFFLINE_AFTER = pd.Timedelta(minutes=10) +_LOG_TIME_RE = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})", re.M) + + +def log_gaps(logs: list[Path], threshold: pd.Timedelta = OFFLINE_AFTER) -> list[tuple[pd.Timestamp, pd.Timestamp]]: + """Periods with no log output for longer than `threshold`, in the bot's + local time: a reboot, a crash, the PC asleep, or a Stop and a Resume.""" + text = [m for p in logs for m in _LOG_TIME_RE.findall(p.read_text(encoding="utf-8", errors="replace"))] + stamps = pd.Series(pd.to_datetime(text, format="%Y-%m-%d %H:%M:%S")).sort_values().reset_index(drop=True) + gap = stamps.diff() > threshold + return [(stamps[i - 1], stamps[i]) for i in gap[gap].index] + + +def _hours(delta: pd.Timedelta) -> str: + hours = delta.total_seconds() / 3600 + return f"{hours:.1f} h" if hours < 48 else f"{hours / 24:.1f} days" def _load_json(path: Path) -> dict: diff --git a/tests/test_panel_api.py b/tests/test_panel_api.py index 283c394..acde5ce 100644 --- a/tests/test_panel_api.py +++ b/tests/test_panel_api.py @@ -135,3 +135,59 @@ def test_reconcile_refuses_until_the_first_candle_closes(client, userdir): r = client.post("/api/paper/sessions/s3/reconcile") assert r.status_code == 409 and "Nothing to replay yet" in r.json()["detail"] assert not (userdir / "paper" / "s3.reconcile.json").exists() + + +def test_resume_needs_a_previous_session(client): + r = client.post("/api/paper/resume") + assert r.status_code == 409 and "no previous" in r.json()["detail"] + + +def test_resume_restarts_the_same_session_with_its_own_pairs(client, monkeypatch): + paper = bot_mod.PAPER_DIR + make_session(paper.parent, "s9", {"strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 100, + "started": "2026-10-01T00:00:00+00:00"}) + (paper / "s9.config.json").write_text(json.dumps({"dry_run": True, "dry_run_wallet": 100, + "exchange": {"key": "", "secret": ""}})) + (paper / "state.json").write_text(json.dumps({"pid": None, "session_id": "s9", "strategy": "TrendRider4h", + "balance": 100, "stopped": "2026-10-05T00:00:00+00:00"})) + launched = [] + monkeypatch.setattr(bot_mod, "_port_open", lambda url: False) + monkeypatch.setattr(bot_mod.subprocess, "run", lambda *a, **k: SimpleNamespace(stdout='{"dry_run": true}', stderr="")) + monkeypatch.setattr(bot_mod.subprocess, "Popen", lambda cmd, **k: launched.append(cmd) or SimpleNamespace(pid=4242)) + + assert client.get("/api/paper/status").json()["can_resume"] is True + state = client.post("/api/paper/resume").json() + assert state["session_id"] == "s9" and state["pid"] == 4242 and len(state["resumed"]) == 1 and "stopped" not in state + # pinned to what the session started with, not today's config.json list + assert json.loads((paper / "s9.config.json").read_text())["exchange"]["pair_whitelist"] == ["SOL/USDT"] + cmd = launched[0] + assert str(paper / "s9.config.json") in cmd and cmd[cmd.index("--strategy") + 1] == "TrendRider4h" + assert client.get("/api/audit").json()["events"][0]["event"] == "paper_session_resumed" + + +def test_resume_refuses_a_live_config(client, monkeypatch): + paper = bot_mod.PAPER_DIR + make_session(paper.parent, "s8", {"strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 100, + "started": "2026-10-01T00:00:00+00:00"}) + (paper / "s8.config.json").write_text(json.dumps({"dry_run": True})) + (paper / "state.json").write_text(json.dumps({"session_id": "s8", "strategy": "TrendRider4h"})) + monkeypatch.setattr(bot_mod, "_port_open", lambda url: False) + monkeypatch.setattr(bot_mod.subprocess, "run", lambda *a, **k: SimpleNamespace(stdout='{"dry_run": false}', stderr="")) + + def no_launch(*a, **k): + raise AssertionError("a bot process was launched") + + monkeypatch.setattr(bot_mod.subprocess, "Popen", no_launch) + r = client.post("/api/paper/resume") + assert r.status_code == 409 and "Refusing" in r.json()["detail"] + + +def test_risk_resume_restarts_trading_not_the_session(client, monkeypatch): + """/api/bot/resume is the risk control (restart entries, clear a halt); + /api/paper/resume restarts a stopped session. They must not be mixed up.""" + calls = [] + monkeypatch.setattr(app_mod.bot, "_call", lambda method, path, **kw: calls.append(path) or {"status": "ok"}) + monkeypatch.setattr(app_mod.bot, "resume_session", lambda: (_ for _ in ()).throw(AssertionError("session resumed"))) + monkeypatch.setattr(bot_mod, "set_halt", lambda *a, **k: None) + assert client.post("/api/bot/resume").status_code == 200 + assert calls == ["start"] diff --git a/tests/test_reconcile.py b/tests/test_reconcile.py index 852fa7a..1ce0c86 100644 --- a/tests/test_reconcile.py +++ b/tests/test_reconcile.py @@ -197,3 +197,15 @@ def bt_trade(pair, opened, closed, reason): assert list(df["is_open"]) == [False, True] assert pd.isna(df.loc[1, "profit_abs"]) and df.loc[0, "profit_abs"] == 10.0 assert meta["end"] == T("2026-10-05 04:00") and meta["timeframe"] == "4h" + + +def test_offline_periods_are_found_in_the_log(tmp_path): + db = tmp_path / "s.sqlite" + make_db(db) + beats = pd.date_range("2026-09-02 10:00", periods=30, freq="1min").append( + pd.date_range("2026-09-02 13:30", periods=30, freq="1min")) # down from 10:29 to 13:30 + (tmp_path / "s.log").write_text("".join(f"{t:%Y-%m-%d %H:%M:%S},123 - freqtrade.worker - INFO - Bot heartbeat.\n" + for t in beats)) + s = session_info(db, pairs=["BTC/USDT"], end="2026-10-01") + assert s.downtime_hours == pytest.approx(3.02, abs=0.01) + assert any("offline 1 time(s)" in w and "3.0 h" in w for w in s.warnings)