📊 telemetry: per-round baseline loss (validator_baseline_loss_by_round)

connito/shared/telemetry.py

@@ -126,6 +126,21 @@ def set_validator_identity(*, hotkey: str, uid: int | None, version: str, netuid
     "validator_baseline_loss",
     "Round baseline loss against this validator's foreground eval set",
 )
+# Per-round baseline loss, labeled by the round it belongs to. The unlabeled
+# gauge above is overwritten every round, so a scraper can only sample it and
+# a cycle's baseline ends up timing-dependent; this labeled family lets the
+# gateway attribute the right baseline to the right round, freeze it after
+# finalize, and naturally exclude a warming-up validator that has no value for
+# a given round. Evicted on the same cutoff as the other per-round families
+# (see evict_round_series_before). The unlabeled family is retained for
+# backward compat during rollout.
+VALIDATOR_BASELINE_LOSS_BY_ROUND = Gauge(
+    "validator_baseline_loss_by_round",
+    "Round baseline loss (this validator's foreground eval set), labeled by the "
+    "round it belongs to. Stable per round; the unlabeled validator_baseline_loss "
+    "is retained for backward compat.",
+    ["round_id"],
+)
 # Numeric ID of the current round, set when `Round.freeze` returns and
 # the round becomes active. Lets aggregators key per-miner score and
 # val_loss readings to a specific round without parsing the round_id
@@ -537,6 +552,26 @@ def note_round_series(round_id: int) -> None:
         pass
 
 
+def set_baseline_loss(round_id: int, baseline_loss: float) -> None:
+    """Publish a round's foreground-eval baseline loss to BOTH the unlabeled
+    gauge (backward compat) and the per-round labeled family.
+
+    Registers the round_id for eviction so the labeled series is pruned on
+    the same cutoff as the other per-round families. Round creation already
+    registers the id via note_round_series; the extra registration here is
+    idempotent (a set) and keeps this helper self-contained.
+
+    Best-effort — telemetry must never block scoring.
+    """
+    try:
+        value = float(baseline_loss)
+        VALIDATOR_BASELINE_LOSS.set(value)
+        note_round_series(int(round_id))
+        VALIDATOR_BASELINE_LOSS_BY_ROUND.labels(round_id=str(int(round_id))).set(value)
+    except Exception:
+        pass
+
+
 def evict_round_series_before(min_round_id: int) -> int:
     """Remove per-round labelsets for every tracked round_id below the
     cutoff. Called from run.py's journal/aggregator prune block with the
@@ -561,6 +596,7 @@ def evict_round_series_before(min_round_id: int) -> int:
                     VALIDATOR_ROUND_MINERS_SCORED,
                     VALIDATOR_ROUND_MINERS_FAILED,
                     VALIDATOR_BG_EVAL_LOCK_LEAK_TOTAL,
+                    VALIDATOR_BASELINE_LOSS_BY_ROUND,
                 ):
                     try:
                         family.remove(rid)

connito/test/test_cycle_consistent_telemetry.py

@@ -268,6 +268,46 @@ def test_finalize_recovery_path_reemits_commits(tmp_path):
     ) == float(rid)
 
 
+# ---------------------------------------------------------------------------
+# Per-round baseline loss
+# ---------------------------------------------------------------------------
+
+def test_set_baseline_loss_sets_both_gauges():
+    rid = 920_100
+    T.set_baseline_loss(rid, 1.8342)
+    # Unlabeled gauge carries the latest value (backward compat).
+    unlabeled = _samples_for(T.VALIDATOR_BASELINE_LOSS, "validator_baseline_loss")
+    assert unlabeled and unlabeled[0].value == 1.8342
+    # Labeled family carries the same value under the round's id.
+    labeled = [
+        s
+        for s in _samples_for(
+            T.VALIDATOR_BASELINE_LOSS_BY_ROUND, "validator_baseline_loss_by_round"
+        )
+        if s.labels["round_id"] == str(rid)
+    ]
+    assert len(labeled) == 1
+    assert labeled[0].value == 1.8342
+    # round_id registered for eviction.
+    assert rid in T._EMITTED_ROUND_IDS
+
+
+def test_set_baseline_loss_labeled_is_stable_per_round():
+    # A second round's baseline does not disturb the first round's labeled
+    # value — the whole point of the per-round label vs the overwritten
+    # unlabeled gauge.
+    T.set_baseline_loss(920_200, 2.0)
+    T.set_baseline_loss(920_724, 3.0)
+    by_round = {
+        s.labels["round_id"]: s.value
+        for s in _samples_for(
+            T.VALIDATOR_BASELINE_LOSS_BY_ROUND, "validator_baseline_loss_by_round"
+        )
+    }
+    assert by_round["920200"] == 2.0
+    assert by_round["920724"] == 3.0
+
+
 # ---------------------------------------------------------------------------
 # Per-round series eviction
 # ---------------------------------------------------------------------------
@@ -277,6 +317,7 @@ def test_evict_round_series_before():
     for rid in (old_rid, new_rid):
         T.note_round_series(rid)
         T.VALIDATOR_ROUND_MINERS_SCORED.labels(round_id=str(rid)).set(5)
+        T.VALIDATOR_BASELINE_LOSS_BY_ROUND.labels(round_id=str(rid)).set(1.8)
     removed = T.evict_round_series_before(new_rid)
     assert removed >= 1
     rids = {
@@ -287,5 +328,14 @@ def test_evict_round_series_before():
     }
     assert str(old_rid) not in rids
     assert str(new_rid) in rids
+    # The baseline-by-round family is evicted on the same cutoff.
+    baseline_rids = {
+        s.labels["round_id"]
+        for s in _samples_for(
+            T.VALIDATOR_BASELINE_LOSS_BY_ROUND, "validator_baseline_loss_by_round"
+        )
+    }
+    assert str(old_rid) not in baseline_rids
+    assert str(new_rid) in baseline_rids
     # Idempotent / KeyError-safe on repeat.
     assert T.evict_round_series_before(new_rid) == 0

connito/validator/evaluator.py

@@ -22,10 +22,10 @@
 )
 from connito.shared.telemetry import (
     EvalFailureReason,
-    VALIDATOR_BASELINE_LOSS,
     VALIDATOR_MINER_VAL_LOSS,
     inc_error,
     inc_eval_failure,
+    set_baseline_loss,
     set_miner_eval_status,
     track_eval_latency,
     track_model_load_latency,
@@ -1028,10 +1028,11 @@ async def evaluate_foreground_round(
     # `delta_loss = max(0, baseline - val_loss)` per miner. Best-effort
     # — Prometheus exposition is purely an observability side-effect
     # and must never block scoring.
-    try:
-        VALIDATOR_BASELINE_LOSS.set(float(baseline_loss))
-    except Exception:
-        pass
+    # Publishes both the unlabeled gauge (backward compat) and the per-round
+    # labeled family so the gateway can attribute this baseline to the exact
+    # round; the labeled series is evicted on the same cutoff as the other
+    # per-round families. Best-effort — never blocks scoring.
+    set_baseline_loss(round_obj.round_id, baseline_loss)
 
     foreground_set = set(round_obj.foreground_uids)
     completed: list[MinerEvalJob] = completed_out if completed_out is not None else []