Resume paper sessions after a restart; flag bot downtime

README.md

@@ -92,6 +92,8 @@ Open **http://127.0.0.1:8090**.
   strategy, press **Start**. The bot trades live Binance prices without sending
   orders. **Reset wallet** starts a fresh session with a new balance; every
   session keeps its own database in `user_data/paper/`, so old results aren't lost.
+  **Resume** restarts the last session after a reboot, crash or Stop, with the
+  same wallet, open trades and pair list. A 4-week paper test survives restarts.
   While it runs, **Open FreqUI** (http://127.0.0.1:8080, log in with the username/password
   from `config.private.json`) shows charts, per-trade detail and more.
 - **Replay:** pick strategy, balance and date range. It simulates that period in

docs/OPERATIONS.md

@@ -231,6 +231,10 @@ The result notes things the replay can't reproduce, so check those first:
 - drawdown halts, which need a manual resume live but resume automatically in backtests
 - strategy code or parameters changed during the session
 - paper trades newer than the latest closed candle
+- time the bot was offline (a reboot, sleep or crash): any stretch of 10+
+  minutes without log output. The replay keeps trading through these gaps, so
+  keep downtime short. After a restart, press **Resume** rather than Reset:
+  Resume continues the same session.
 
 **If it fails.** Don't loosen the targets. Find out why the two diverged. If
 the backtest assumptions are wrong (for example, real fills are consistently

panel/app.py

@@ -100,6 +100,14 @@ def paper_reset(req: PaperReset):
         raise HTTPException(409, str(e))
 
 
+@app.post("/api/paper/resume")
+def paper_resume():
+    try:
+        return bot.resume_session()
+    except PaperBotError as e:
+        raise HTTPException(409, str(e))
+
+
 @app.get("/api/mode")
 def mode():
     return {"monitor": MONITOR_MODE, "version": __version__}

panel/bot.py

@@ -29,7 +29,7 @@
 
 from panel.common import FREQTRADE, ROOT, USERDIR, config_args, load_json, merged_config
 from ridgeline import audit
-from ridgeline.reconcile import strategy_fingerprint
+from ridgeline.reconcile import session_info, strategy_fingerprint
 from ridgeline.risk import RiskLimits, build_state, limits_summary, load_halt, set_halt
 
 MONITOR_MODE = os.environ.get("PANEL_MODE", "").lower() == "monitor"
@@ -87,11 +87,9 @@ def _our_process(self) -> Optional[psutil.Process]:
 
     # --- actions ---
 
-    def start(self, balance: float, strategy: str) -> dict:
+    def _preflight(self) -> None:
         if MONITOR_MODE:
             raise PaperBotError("Process control is disabled in monitor mode (Docker manages the bot).")
-        if balance <= 0:
-            raise PaperBotError("Starting balance must be positive.")
         if self._our_process():
             raise PaperBotError("A paper session is already running. Stop or reset it first.")
         if _port_open(self.base_url):
@@ -100,30 +98,84 @@ def start(self, balance: float, strategy: str) -> dict:
                 "started by hand). Stop it first - the panel will not kill processes it didn't start."
             )
 
+    def start(self, balance: float, strategy: str) -> dict:
+        if balance <= 0:
+            raise PaperBotError("Starting balance must be positive.")
+        self._preflight()
+
         session_id = datetime.now().strftime("%Y%m%d-%H%M%S")
+        cfg = merged_config()
+        pairs = cfg.get("exchange", {}).get("pair_whitelist", [])
         session_cfg = PAPER_DIR / f"{session_id}.config.json"
         session_cfg.write_text(json.dumps({
             "dry_run": True,
             "dry_run_wallet": balance,
             "db_url": f"sqlite:///user_data/paper/{session_id}.sqlite",
             "bot_name": f"paper-{session_id}",
-            "exchange": {"key": "", "secret": ""},
+            # Pin the pair list: config.json's changes monthly, and a session
+            # resumed after a restart must keep trading what it started with.
+            "exchange": {"key": "", "secret": "", "pair_whitelist": pairs},
         }, indent=2))
         args = [*config_args(), "-c", str(session_cfg)]
         self._verify_dry_run(args)
         # What this session runs, for reconciling it with a replay later:
         # config.json's pair list changes monthly, and code can change mid-session.
-        cfg = merged_config()
         (PAPER_DIR / f"{session_id}.meta.json").write_text(json.dumps({
             "session_id": session_id,
             "strategy": strategy,
             "balance": balance,
-            "pairs": cfg.get("exchange", {}).get("pair_whitelist", []),
+            "pairs": pairs,
             "max_open_trades": cfg.get("max_open_trades"),
             "started": datetime.now(timezone.utc).isoformat(timespec="seconds"),
             "code": strategy_fingerprint(strategy),
         }, indent=2))
 
+        state = {
+            "pid": self._spawn(session_id, args, strategy),
+            "session_id": session_id,
+            "strategy": strategy,
+            "balance": balance,
+            "started": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        }
+        self._save_state(state)
+        audit.append("paper_session_started", "panel",
+                     {"session_id": session_id, "strategy": strategy, "balance": balance})
+        return state
+
+    def resume_session(self) -> dict:
+        """Restart the last session's bot on the same database, after a reboot,
+        a crash or a Stop. Open trades, the wallet and the trade history carry
+        on, so a weeks-long paper test isn't lost to a restart."""
+        self._preflight()
+        state = self._state()
+        session_id = state.get("session_id")
+        if not session_id:
+            raise PaperBotError("There's no previous paper session to resume.")
+        session_cfg = PAPER_DIR / f"{session_id}.config.json"
+        db = PAPER_DIR / f"{session_id}.sqlite"
+        if not session_cfg.exists() or not db.exists():
+            raise PaperBotError(f"Session {session_id}'s files are missing. Start a new session instead.")
+        try:
+            info = session_info(db)
+        except ValueError as e:
+            raise PaperBotError(f"Can't resume session {session_id}: {e}") from e
+        cfg = load_json(session_cfg, {}) or {}
+        if not cfg.get("exchange", {}).get("pair_whitelist"):
+            # Sessions from before pairs were pinned: keep what they traded,
+            # not whatever config.json lists today.
+            cfg.setdefault("exchange", {}).update(key="", secret="", pair_whitelist=info.pairs)
+            session_cfg.write_text(json.dumps(cfg, indent=2))
+        args = [*config_args(), "-c", str(session_cfg)]
+        self._verify_dry_run(args)
+        strategy = state.get("strategy") or info.strategy
+        state.update(pid=self._spawn(session_id, args, strategy), strategy=strategy,
+                     resumed=[*state.get("resumed", []), datetime.now(timezone.utc).isoformat(timespec="seconds")])
+        state.pop("stopped", None)
+        self._save_state(state)
+        audit.append("paper_session_resumed", "panel", {"session_id": session_id, "strategy": strategy})
+        return state
+
+    def _spawn(self, session_id: str, args: list[str], strategy: str) -> int:
         log_path = PAPER_DIR / f"{session_id}.log"
         cmd = [*FREQTRADE, "trade", *args, "--strategy", strategy, "--logfile", str(log_path)]
         popen_kw = dict(cwd=ROOT, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
@@ -138,17 +190,7 @@ def start(self, balance: float, strategy: str) -> dict:
                 proc = subprocess.Popen(cmd, creationflags=flags, **popen_kw)
         else:
             proc = subprocess.Popen(cmd, start_new_session=True, **popen_kw)
-        state = {
-            "pid": proc.pid,
-            "session_id": session_id,
-            "strategy": strategy,
-            "balance": balance,
-            "started": datetime.now(timezone.utc).isoformat(timespec="seconds"),
-        }
-        self._save_state(state)
-        audit.append("paper_session_started", "panel",
-                     {"session_id": session_id, "strategy": strategy, "balance": balance})
-        return state
+        return proc.pid
 
     def _verify_dry_run(self, args: list[str]) -> None:
         out = subprocess.run([*FREQTRADE, "show-config", *args], cwd=ROOT,
@@ -289,6 +331,8 @@ def status(self) -> dict:
             "api_up": False,
             "session": state if ours else None,
             "last_session": state if not ours and state else None,
+            "can_resume": (not ours and not listening and not MONITOR_MODE and bool(state.get("session_id"))
+                           and (PAPER_DIR / f"{state.get('session_id')}.sqlite").exists()),
             "freqtrade_ui": self.base_url,
         }
         if not listening:

panel/static/app.js

@@ -130,6 +130,7 @@ async function refreshPaper() {
 
   $("#paper-start").disabled = paperBusy || st.managed || st.external || monitorMode;
   $("#paper-stop").disabled = paperBusy || !st.managed || monitorMode;
+  $("#paper-resume").disabled = paperBusy || !st.can_resume;
   $("#paper-reset").disabled = paperBusy || st.external || monitorMode;
   ["#risk-pause", "#risk-resume", "#risk-flatten"].forEach((id) => ($(id).disabled = paperBusy || !st.api_up));
   if (st.api_up) refreshRisk(); else $("#risk-meters").innerHTML = `<p class="muted small">Available while a bot is running.</p>`;
@@ -196,6 +197,7 @@ $("#paper-form").addEventListener("submit", (e) => {
   paperAction(() => api("/api/paper/start", { method: "POST", body: { balance: +$("#paper-balance").value, strategy: $("#paper-strategy").value } }));
 });
 $("#paper-stop").addEventListener("click", () => paperAction(() => api("/api/paper/stop", { method: "POST" })));
+$("#paper-resume").addEventListener("click", () => paperAction(() => api("/api/paper/resume", { method: "POST" })));
 $("#paper-reset").addEventListener("click", () => {
   const bal = +$("#paper-balance").value;
   if (!confirm(`Start a fresh paper session with ${bal} USDT? The current session's trades are kept in its own archive file.`)) return;

panel/static/index.html

@@ -55,6 +55,7 @@ <h2>Fake wallet</h2>
           <div class="btn-row">
             <button type="submit" class="btn btn-primary" id="paper-start">Start</button>
             <button type="button" class="btn" id="paper-stop">Stop</button>
+            <button type="button" class="btn" id="paper-resume" title="Restart the last session's bot on the same wallet and trade history, e.g. after a reboot.">Resume</button>
             <button type="button" class="btn btn-warn" id="paper-reset" title="Stop, then start a fresh session with a new wallet. Old sessions are kept.">Reset wallet</button>
           </div>
         </form>

ridgeline/reconcile.py

@@ -75,11 +75,12 @@ class SessionInfo:
     end: pd.Timestamp
     max_open_trades: Optional[int] = None
     warnings: list[str] = field(default_factory=list)
+    downtime_hours: float = 0.0
 
     def to_dict(self) -> dict:
         return {"session_id": self.session_id, "strategy": self.strategy, "pairs": self.pairs,
                 "balance": self.balance, "start": self.start.isoformat(), "end": self.end.isoformat(),
-                "max_open_trades": self.max_open_trades}
+                "max_open_trades": self.max_open_trades, "downtime_hours": self.downtime_hours}
 
 
 def strategy_fingerprint(strategy: str, strategy_dir: Path = STRATEGY_DIR) -> dict[str, Optional[str]]:
@@ -154,8 +155,36 @@ def session_info(db: Path, *, log: Optional[Path] = None, strategy: Optional[str
             warnings.append(f"Changed since the session started: {', '.join(changed)}. "
                             "The replay runs the current version, so some differences are expected.")
 
+    gaps = log_gaps(logs)
+    downtime = sum((b - a for a, b in gaps), pd.Timedelta(0))
+    if gaps:
+        longest = max(b - a for a, b in gaps)
+        warnings.append(f"The bot was offline {len(gaps)} time(s), {_hours(downtime)} in total (longest "
+                        f"{_hours(longest)}, from {gaps[0][0]:%Y-%m-%d %H:%M} local time on). The replay trades through "
+                        "those gaps, so trades it takes while the bot was down are missing from paper.")
+
     return SessionInfo(session_id=db.stem, db=db, strategy=strategy, pairs=list(pairs), balance=float(balance),
-                       start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings)
+                       start=start, end=end, max_open_trades=meta.get("max_open_trades"), warnings=warnings,
+                       downtime_hours=round(downtime.total_seconds() / 3600, 2))
+
+
+# A running bot logs a heartbeat every minute; silence longer than this means it was down.
+OFFLINE_AFTER = pd.Timedelta(minutes=10)
+_LOG_TIME_RE = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})", re.M)
+
+
+def log_gaps(logs: list[Path], threshold: pd.Timedelta = OFFLINE_AFTER) -> list[tuple[pd.Timestamp, pd.Timestamp]]:
+    """Periods with no log output for longer than `threshold`, in the bot's
+    local time: a reboot, a crash, the PC asleep, or a Stop and a Resume."""
+    text = [m for p in logs for m in _LOG_TIME_RE.findall(p.read_text(encoding="utf-8", errors="replace"))]
+    stamps = pd.Series(pd.to_datetime(text, format="%Y-%m-%d %H:%M:%S")).sort_values().reset_index(drop=True)
+    gap = stamps.diff() > threshold
+    return [(stamps[i - 1], stamps[i]) for i in gap[gap].index]
+
+
+def _hours(delta: pd.Timedelta) -> str:
+    hours = delta.total_seconds() / 3600
+    return f"{hours:.1f} h" if hours < 48 else f"{hours / 24:.1f} days"
 
 
 def _load_json(path: Path) -> dict:

tests/test_panel_api.py

@@ -135,3 +135,59 @@ def test_reconcile_refuses_until_the_first_candle_closes(client, userdir):
     r = client.post("/api/paper/sessions/s3/reconcile")
     assert r.status_code == 409 and "Nothing to replay yet" in r.json()["detail"]
     assert not (userdir / "paper" / "s3.reconcile.json").exists()
+
+
+def test_resume_needs_a_previous_session(client):
+    r = client.post("/api/paper/resume")
+    assert r.status_code == 409 and "no previous" in r.json()["detail"]
+
+
+def test_resume_restarts_the_same_session_with_its_own_pairs(client, monkeypatch):
+    paper = bot_mod.PAPER_DIR
+    make_session(paper.parent, "s9", {"strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 100,
+                                      "started": "2026-10-01T00:00:00+00:00"})
+    (paper / "s9.config.json").write_text(json.dumps({"dry_run": True, "dry_run_wallet": 100,
+                                                      "exchange": {"key": "", "secret": ""}}))
+    (paper / "state.json").write_text(json.dumps({"pid": None, "session_id": "s9", "strategy": "TrendRider4h",
+                                                  "balance": 100, "stopped": "2026-10-05T00:00:00+00:00"}))
+    launched = []
+    monkeypatch.setattr(bot_mod, "_port_open", lambda url: False)
+    monkeypatch.setattr(bot_mod.subprocess, "run", lambda *a, **k: SimpleNamespace(stdout='{"dry_run": true}', stderr=""))
+    monkeypatch.setattr(bot_mod.subprocess, "Popen", lambda cmd, **k: launched.append(cmd) or SimpleNamespace(pid=4242))
+
+    assert client.get("/api/paper/status").json()["can_resume"] is True
+    state = client.post("/api/paper/resume").json()
+    assert state["session_id"] == "s9" and state["pid"] == 4242 and len(state["resumed"]) == 1 and "stopped" not in state
+    # pinned to what the session started with, not today's config.json list
+    assert json.loads((paper / "s9.config.json").read_text())["exchange"]["pair_whitelist"] == ["SOL/USDT"]
+    cmd = launched[0]
+    assert str(paper / "s9.config.json") in cmd and cmd[cmd.index("--strategy") + 1] == "TrendRider4h"
+    assert client.get("/api/audit").json()["events"][0]["event"] == "paper_session_resumed"
+
+
+def test_resume_refuses_a_live_config(client, monkeypatch):
+    paper = bot_mod.PAPER_DIR
+    make_session(paper.parent, "s8", {"strategy": "TrendRider4h", "pairs": ["SOL/USDT"], "balance": 100,
+                                      "started": "2026-10-01T00:00:00+00:00"})
+    (paper / "s8.config.json").write_text(json.dumps({"dry_run": True}))
+    (paper / "state.json").write_text(json.dumps({"session_id": "s8", "strategy": "TrendRider4h"}))
+    monkeypatch.setattr(bot_mod, "_port_open", lambda url: False)
+    monkeypatch.setattr(bot_mod.subprocess, "run", lambda *a, **k: SimpleNamespace(stdout='{"dry_run": false}', stderr=""))
+
+    def no_launch(*a, **k):
+        raise AssertionError("a bot process was launched")
+
+    monkeypatch.setattr(bot_mod.subprocess, "Popen", no_launch)
+    r = client.post("/api/paper/resume")
+    assert r.status_code == 409 and "Refusing" in r.json()["detail"]
+
+
+def test_risk_resume_restarts_trading_not_the_session(client, monkeypatch):
+    """/api/bot/resume is the risk control (restart entries, clear a halt);
+    /api/paper/resume restarts a stopped session. They must not be mixed up."""
+    calls = []
+    monkeypatch.setattr(app_mod.bot, "_call", lambda method, path, **kw: calls.append(path) or {"status": "ok"})
+    monkeypatch.setattr(app_mod.bot, "resume_session", lambda: (_ for _ in ()).throw(AssertionError("session resumed")))
+    monkeypatch.setattr(bot_mod, "set_halt", lambda *a, **k: None)
+    assert client.post("/api/bot/resume").status_code == 200
+    assert calls == ["start"]

tests/test_reconcile.py

@@ -197,3 +197,15 @@ def bt_trade(pair, opened, closed, reason):
     assert list(df["is_open"]) == [False, True]
     assert pd.isna(df.loc[1, "profit_abs"]) and df.loc[0, "profit_abs"] == 10.0
     assert meta["end"] == T("2026-10-05 04:00") and meta["timeframe"] == "4h"
+
+
+def test_offline_periods_are_found_in_the_log(tmp_path):
+    db = tmp_path / "s.sqlite"
+    make_db(db)
+    beats = pd.date_range("2026-09-02 10:00", periods=30, freq="1min").append(
+        pd.date_range("2026-09-02 13:30", periods=30, freq="1min"))   # down from 10:29 to 13:30
+    (tmp_path / "s.log").write_text("".join(f"{t:%Y-%m-%d %H:%M:%S},123 - freqtrade.worker - INFO - Bot heartbeat.\n"
+                                            for t in beats))
+    s = session_info(db, pairs=["BTC/USDT"], end="2026-10-01")
+    assert s.downtime_hours == pytest.approx(3.02, abs=0.01)
+    assert any("offline 1 time(s)" in w and "3.0 h" in w for w in s.warnings)