📊 telemetry: restore round-level gauges on journal recovery (schema v3)

connito/test/test_cycle_consistent_telemetry.py

@@ -155,6 +155,78 @@ def test_commit_map_from_checkpoints_skips_incomplete():
     assert m == {1: ("a/r", "v1")}
 
 
+# ---------------------------------------------------------------------------
+# Journal v3: roster_size / lifecycle_step for round-gauge recovery
+# ---------------------------------------------------------------------------
+
+def test_journal_v3_roundtrips_roster_and_step(tmp_path):
+    j = RJ.RoundJournal(
+        round_id=555,
+        scored_uids=(1, 2, 3),
+        failed_uids=(4,),
+        roster_size=165,
+        lifecycle_step=3,
+        finalized=True,
+    )
+    p = tmp_path / "round_555.json"
+    RJ.write_atomic(p, j)
+    loaded = RJ.load(p)
+    assert loaded is not None
+    assert loaded.schema_version == 3
+    assert loaded.roster_size == 165
+    assert loaded.lifecycle_step == 3
+
+
+def test_journal_v2_loads_with_zero_roster(tmp_path):
+    # A v2 file (no roster_size/lifecycle_step) must still load, defaulting
+    # both to 0 so recovery degrades gracefully rather than crashing.
+    v2 = {
+        "round_id": 42, "schema_version": 2,
+        "scored_uids": [1, 2], "failed_uids": [3],
+        "uid_to_commit": {}, "finalized": False,
+    }
+    p = tmp_path / "round_42.json"
+    p.write_text(json.dumps(v2), encoding="utf-8")
+    loaded = RJ.load(p)
+    assert loaded is not None
+    assert loaded.roster_size == 0
+    assert loaded.lifecycle_step == 0
+
+
+def test_recovery_round_carries_v3_fields(tmp_path):
+    j = RJ.RoundJournal(round_id=777, roster_size=100, lifecycle_step=2)
+    stub = RJ._RecoveryRound.from_journal(j, tmp_path / "round_777.json")
+    assert stub.roster_size == 100
+    assert stub.lifecycle_step == 2
+
+
+def test_finalize_preserves_roster_in_journal_rewrite(tmp_path):
+    # finalize re-writes the journal with finalized=True; the v3 fields must
+    # survive that round-trip (they feed the round-gauge restore next boot).
+    rid = 8_888
+    jpath = tmp_path / f"round_{rid}.json"
+    RJ.write_atomic(
+        jpath,
+        RJ.RoundJournal(
+            round_id=rid,
+            uid_to_hotkey={9201: "hk", 9202: "hk2"},
+            scores={9201: 2.5},
+            scored_uids=(9201,),
+            failed_uids=(9202,),
+            roster_size=165,
+            lifecycle_step=3,
+        ),
+    )
+    stub = RJ._RecoveryRound.from_journal(RJ.load(jpath), jpath)
+    finalize_round_scores(
+        round_obj=stub, score_aggregator=_FakeAggregator(), score_path=None,
+    )
+    reloaded = RJ.load(jpath)
+    assert reloaded.finalized is True
+    assert reloaded.roster_size == 165
+    assert reloaded.lifecycle_step == 3
+
+
 # ---------------------------------------------------------------------------
 # finalize_round_scores emission (live-shaped round + recovery path)
 # ---------------------------------------------------------------------------

connito/validator/evaluator.py

@@ -347,6 +347,18 @@ def finalize_round_scores(
                     uid_to_commit=_rj.commit_map_from_checkpoints(
                         getattr(round_obj, "uid_to_chain_checkpoint", None) or {}
                     ),
+                    # v3 round-level gauge inputs. A live Round exposes
+                    # foreground/background uids; the recovery stub carries
+                    # `roster_size` forward from the journal it was hydrated
+                    # from — so a re-finalize preserves them.
+                    roster_size=int(
+                        getattr(round_obj, "roster_size", 0)
+                        or (
+                            len(getattr(round_obj, "foreground_uids", ()))
+                            + len(getattr(round_obj, "background_uids", ()))
+                        )
+                    ),
+                    lifecycle_step=int(getattr(round_obj, "lifecycle_step", 0)),
                     finalized=True,
                 ),
             )

connito/validator/round.py

@@ -95,6 +95,11 @@ class Round:
     freeze_zero_uids: set[int] = field(default_factory=set)
     freeze_zero_hotkeys: dict[int, str] = field(default_factory=dict)
     weights_submitted: bool = False
+    # Last live lifecycle step this round reached (set by run.py alongside
+    # the VALIDATOR_ROUND_LIFECYCLE_STEP gauge: 0 freeze / 2 post-foreground
+    # / 3 eval-window). Persisted to the journal so startup recovery can
+    # restore the round-level gauges that only the live loop writes.
+    lifecycle_step: int = 0
 
     # Round-group construction scheme (gated by
     # `config.evaluation.enable_round_group_construction`). All default
@@ -587,6 +592,8 @@ def _journal_snapshot_locked(self) -> dict | None:
             "freeze_zero_uids": tuple(sorted(self.freeze_zero_uids)),
             "freeze_zero_hotkeys": dict(self.freeze_zero_hotkeys),
             "uid_to_commit": commit_map_from_checkpoints(self.uid_to_chain_checkpoint),
+            "roster_size": len(self.foreground_uids) + len(self.background_uids),
+            "lifecycle_step": int(self.lifecycle_step),
             "finalized": False,
         }
 

connito/validator/round_journal.py

@@ -27,9 +27,14 @@
 
 # v2 adds `uid_to_commit` (uid -> (hf_repo_id, hf_revision)) so the
 # journal-recovery finalize can re-publish evaluated-commit telemetry.
-# `from_json` accepts v1 files (missing map -> empty) so leftover journals
-# written by an older build still recover.
-SCHEMA_VERSION = 2
+# v3 adds `roster_size` + `lifecycle_step` so recovery can also restore the
+# round-level gauges (validator_round_miners_{scored,pending,failed},
+# lifecycle_step, current_round_id) — those are written only by the live
+# eval workers, so without persistence the dashboard's "Evaluated N of M"
+# column blanks for a full cycle after every restart.
+# `from_json` accepts v1/v2 files (missing fields default) so leftover
+# journals written by an older build still recover.
+SCHEMA_VERSION = 3
 JOURNAL_DIR_NAME = "round_journal"
 JOURNAL_FILENAME_PREFIX = "round_"
 JOURNAL_FILENAME_SUFFIX = ".json"
@@ -59,6 +64,14 @@ class RoundJournal:
     # v2: uid -> (hf_repo_id, hf_revision) evaluated for the round. Only
     # uids whose chain checkpoint carried BOTH values are recorded.
     uid_to_commit: dict[int, tuple[str, str]] = field(default_factory=dict)
+    # v3: round-level gauge inputs. `roster_size` = len(foreground_uids) +
+    # len(background_uids) at freeze; `lifecycle_step` = the last live
+    # lifecycle step the round reached (0 freeze / 2 post-foreground /
+    # 3 eval-window). Both let startup recovery restore the round-level
+    # gauges. Default 0 for v1/v2 journals (recovery then leaves pending at
+    # 0 rather than inventing a roster).
+    roster_size: int = 0
+    lifecycle_step: int = 0
     finalized: bool = False
     schema_version: int = SCHEMA_VERSION
 
@@ -106,6 +119,8 @@ def from_json(cls, data: str) -> "RoundJournal":
                 for k, v in raw.get("uid_to_commit", {}).items()
                 if isinstance(v, (list, tuple)) and len(v) == 2
             },
+            roster_size=int(raw.get("roster_size", 0)),
+            lifecycle_step=int(raw.get("lifecycle_step", 0)),
             finalized=bool(raw.get("finalized", False)),
             schema_version=version,
         )
@@ -218,6 +233,10 @@ class _RecoveryRound:
     # `uid_to_commit` map (empty for v1 journals) so a recovered finalize
     # re-publishes evaluated-commit telemetry too.
     uid_to_chain_checkpoint: dict[int, object] = field(default_factory=dict)
+    # v3: carried through so the finalize re-write (finalized=True) preserves
+    # them, and so the recovery pass can restore the round-level gauges.
+    roster_size: int = 0
+    lifecycle_step: int = 0
     _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
 
     @classmethod
@@ -238,6 +257,8 @@ def from_journal(cls, journal: "RoundJournal", journal_path: str | os.PathLike)
                 int(uid): SimpleNamespace(hf_repo_id=repo, hf_revision=rev)
                 for uid, (repo, rev) in journal.uid_to_commit.items()
             },
+            roster_size=int(journal.roster_size),
+            lifecycle_step=int(journal.lifecycle_step),
         )
 
     def processed_uids_snapshot(self) -> tuple[set[int], set[int]]:

connito/validator/run.py

@@ -183,6 +183,9 @@ def validate_hf_distribution_config(config: ValidatorConfig) -> tuple[str | None
     VALIDATOR_HEARTBEAT_TOTAL,
     VALIDATOR_MINER_WEIGHT_SUBMITTED,
     VALIDATOR_ROUND_LIFECYCLE_STEP,
+    VALIDATOR_ROUND_MINERS_FAILED,
+    VALIDATOR_ROUND_MINERS_PENDING,
+    VALIDATOR_ROUND_MINERS_SCORED,
     SystemStatePoller,
     set_miner_assignment_role,
     set_miner_cohort_group,
@@ -853,6 +856,34 @@ def run(rank: int, world_size: int, config: ValidatorConfig, pkg_version: str =
                     score_aggregator=score_aggregator,
                     score_path=score_path,
                 )
+                # Restore the round-level gauges the live eval workers write
+                # (validator_round_miners_{scored,pending,failed}, lifecycle
+                # step, current_round_id). finalize replays the per-miner
+                # families, but these are never touched by recovery, so the
+                # dashboard's "Evaluated N of M" column would otherwise stay
+                # blank for a full cycle after every restart. roster_size is
+                # 0 for pre-v3 journals — pending clamps to 0 rather than
+                # inventing a denominator. Journals are scanned ascending, so
+                # current_round_id lands on the most recent recovered round.
+                try:
+                    _rscored = len(_journal.scored_uids)
+                    _rfailed = len(_journal.failed_uids)
+                    _rpending = max(0, int(_journal.roster_size) - _rscored - _rfailed)
+                    note_round_series(_journal.round_id)
+                    _rid_label = str(_journal.round_id)
+                    VALIDATOR_ROUND_MINERS_SCORED.labels(round_id=_rid_label).set(_rscored)
+                    VALIDATOR_ROUND_MINERS_FAILED.labels(round_id=_rid_label).set(_rfailed)
+                    VALIDATOR_ROUND_MINERS_PENDING.labels(round_id=_rid_label).set(_rpending)
+                    if _journal.lifecycle_step:
+                        VALIDATOR_ROUND_LIFECYCLE_STEP.labels(
+                            round_id=_rid_label
+                        ).set(int(_journal.lifecycle_step))
+                    VALIDATOR_CURRENT_ROUND_ID.set(float(_journal.round_id))
+                except Exception as _e:
+                    logger.warning(
+                        "Startup recovery: round-gauge restore failed",
+                        round_id=_journal.round_id, error=str(_e),
+                    )
                 _recovered += 1
                 logger.info(
                     "Startup recovery: finalized journal",
@@ -1416,6 +1447,7 @@ def run(rank: int, world_size: int, config: ValidatorConfig, pkg_version: str =
             download_window_closed.clear()
             try:
                 note_round_series(new_round.round_id)
+                new_round.lifecycle_step = 0
                 VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(new_round.round_id)).set(0)
             except Exception:
                 pass
@@ -1496,6 +1528,7 @@ async def _bounded_foreground_eval():
                 eval_worker.set_eval_base_model(copy.deepcopy(global_model))
             try:
                 note_round_series(new_round.round_id)
+                new_round.lifecycle_step = 2
                 VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(new_round.round_id)).set(2)
             except Exception:
                 pass
@@ -1728,6 +1761,7 @@ async def _bounded_foreground_eval():
             eval_window_active.set()
             try:
                 note_round_series(new_round.round_id)
+                new_round.lifecycle_step = 3
                 VALIDATOR_ROUND_LIFECYCLE_STEP.labels(round_id=str(new_round.round_id)).set(3)
             except Exception:
                 pass