From 15ba5d99a79c8d09744df3fed5c7c13d84331a19 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 24 Jul 2026 16:54:05 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=8A=20telemetry:=20per-round=20baselin?= =?UTF-8?q?e=20loss=20(validator=5Fbaseline=5Floss=5Fby=5Fround)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard needs a stable per-cycle baseline, but validator_baseline_loss is a single unlabeled gauge overwritten each round, so the gateway can only sample it and a cycle's baseline ends up timing-dependent. Add validator_baseline_loss_by_round{round_id} labeled exactly like the round-attribution families. A new best-effort set_baseline_loss(round_id, value) helper publishes BOTH the unlabeled gauge (backward compat during rollout) and the labeled one, and registers the round for eviction; the foreground-eval baseline site calls it in place of the prior inline set. The labeled family is added to evict_round_series_before so its stale labelsets prune on the same cutoff as the other per-round series. Emitting per-round lets the gateway attribute the right baseline to the right cycle, freeze it after finalize, and naturally exclude a warming-up validator that has no value for that round (the gateway averages across validators, dropping 0/absent). Tests: helper sets both gauges with the round's id; labeled value is stable per round; eviction test extended to cover the baseline family. Full telemetry suite green in the stable image. Co-Authored-By: Claude Fable 5 --- connito/shared/telemetry.py | 36 +++++++++++++ .../test/test_cycle_consistent_telemetry.py | 50 +++++++++++++++++++ connito/validator/evaluator.py | 11 ++-- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/connito/shared/telemetry.py b/connito/shared/telemetry.py index c57bcf7..6aa6623 100644 --- a/connito/shared/telemetry.py +++ b/connito/shared/telemetry.py @@ -126,6 +126,21 @@ def set_validator_identity(*, hotkey: str, uid: int | None, version: str, netuid "validator_baseline_loss", "Round baseline loss against this validator's foreground eval set", ) +# Per-round baseline loss, labeled by the round it belongs to. The unlabeled +# gauge above is overwritten every round, so a scraper can only sample it and +# a cycle's baseline ends up timing-dependent; this labeled family lets the +# gateway attribute the right baseline to the right round, freeze it after +# finalize, and naturally exclude a warming-up validator that has no value for +# a given round. Evicted on the same cutoff as the other per-round families +# (see evict_round_series_before). The unlabeled family is retained for +# backward compat during rollout. +VALIDATOR_BASELINE_LOSS_BY_ROUND = Gauge( + "validator_baseline_loss_by_round", + "Round baseline loss (this validator's foreground eval set), labeled by the " + "round it belongs to. Stable per round; the unlabeled validator_baseline_loss " + "is retained for backward compat.", + ["round_id"], +) # Numeric ID of the current round, set when `Round.freeze` returns and # the round becomes active. Lets aggregators key per-miner score and # val_loss readings to a specific round without parsing the round_id @@ -537,6 +552,26 @@ def note_round_series(round_id: int) -> None: pass +def set_baseline_loss(round_id: int, baseline_loss: float) -> None: + """Publish a round's foreground-eval baseline loss to BOTH the unlabeled + gauge (backward compat) and the per-round labeled family. + + Registers the round_id for eviction so the labeled series is pruned on + the same cutoff as the other per-round families. Round creation already + registers the id via note_round_series; the extra registration here is + idempotent (a set) and keeps this helper self-contained. + + Best-effort — telemetry must never block scoring. + """ + try: + value = float(baseline_loss) + VALIDATOR_BASELINE_LOSS.set(value) + note_round_series(int(round_id)) + VALIDATOR_BASELINE_LOSS_BY_ROUND.labels(round_id=str(int(round_id))).set(value) + except Exception: + pass + + def evict_round_series_before(min_round_id: int) -> int: """Remove per-round labelsets for every tracked round_id below the cutoff. Called from run.py's journal/aggregator prune block with the @@ -561,6 +596,7 @@ def evict_round_series_before(min_round_id: int) -> int: VALIDATOR_ROUND_MINERS_SCORED, VALIDATOR_ROUND_MINERS_FAILED, VALIDATOR_BG_EVAL_LOCK_LEAK_TOTAL, + VALIDATOR_BASELINE_LOSS_BY_ROUND, ): try: family.remove(rid) diff --git a/connito/test/test_cycle_consistent_telemetry.py b/connito/test/test_cycle_consistent_telemetry.py index 0673030..e563dc1 100644 --- a/connito/test/test_cycle_consistent_telemetry.py +++ b/connito/test/test_cycle_consistent_telemetry.py @@ -268,6 +268,46 @@ def test_finalize_recovery_path_reemits_commits(tmp_path): ) == float(rid) +# --------------------------------------------------------------------------- +# Per-round baseline loss +# --------------------------------------------------------------------------- + +def test_set_baseline_loss_sets_both_gauges(): + rid = 920_100 + T.set_baseline_loss(rid, 1.8342) + # Unlabeled gauge carries the latest value (backward compat). + unlabeled = _samples_for(T.VALIDATOR_BASELINE_LOSS, "validator_baseline_loss") + assert unlabeled and unlabeled[0].value == 1.8342 + # Labeled family carries the same value under the round's id. + labeled = [ + s + for s in _samples_for( + T.VALIDATOR_BASELINE_LOSS_BY_ROUND, "validator_baseline_loss_by_round" + ) + if s.labels["round_id"] == str(rid) + ] + assert len(labeled) == 1 + assert labeled[0].value == 1.8342 + # round_id registered for eviction. + assert rid in T._EMITTED_ROUND_IDS + + +def test_set_baseline_loss_labeled_is_stable_per_round(): + # A second round's baseline does not disturb the first round's labeled + # value — the whole point of the per-round label vs the overwritten + # unlabeled gauge. + T.set_baseline_loss(920_200, 2.0) + T.set_baseline_loss(920_724, 3.0) + by_round = { + s.labels["round_id"]: s.value + for s in _samples_for( + T.VALIDATOR_BASELINE_LOSS_BY_ROUND, "validator_baseline_loss_by_round" + ) + } + assert by_round["920200"] == 2.0 + assert by_round["920724"] == 3.0 + + # --------------------------------------------------------------------------- # Per-round series eviction # --------------------------------------------------------------------------- @@ -277,6 +317,7 @@ def test_evict_round_series_before(): for rid in (old_rid, new_rid): T.note_round_series(rid) T.VALIDATOR_ROUND_MINERS_SCORED.labels(round_id=str(rid)).set(5) + T.VALIDATOR_BASELINE_LOSS_BY_ROUND.labels(round_id=str(rid)).set(1.8) removed = T.evict_round_series_before(new_rid) assert removed >= 1 rids = { @@ -287,5 +328,14 @@ def test_evict_round_series_before(): } assert str(old_rid) not in rids assert str(new_rid) in rids + # The baseline-by-round family is evicted on the same cutoff. + baseline_rids = { + s.labels["round_id"] + for s in _samples_for( + T.VALIDATOR_BASELINE_LOSS_BY_ROUND, "validator_baseline_loss_by_round" + ) + } + assert str(old_rid) not in baseline_rids + assert str(new_rid) in baseline_rids # Idempotent / KeyError-safe on repeat. assert T.evict_round_series_before(new_rid) == 0 diff --git a/connito/validator/evaluator.py b/connito/validator/evaluator.py index df2f5f9..93a9467 100644 --- a/connito/validator/evaluator.py +++ b/connito/validator/evaluator.py @@ -22,10 +22,10 @@ ) from connito.shared.telemetry import ( EvalFailureReason, - VALIDATOR_BASELINE_LOSS, VALIDATOR_MINER_VAL_LOSS, inc_error, inc_eval_failure, + set_baseline_loss, set_miner_eval_status, track_eval_latency, track_model_load_latency, @@ -1028,10 +1028,11 @@ async def evaluate_foreground_round( # `delta_loss = max(0, baseline - val_loss)` per miner. Best-effort # — Prometheus exposition is purely an observability side-effect # and must never block scoring. - try: - VALIDATOR_BASELINE_LOSS.set(float(baseline_loss)) - except Exception: - pass + # Publishes both the unlabeled gauge (backward compat) and the per-round + # labeled family so the gateway can attribute this baseline to the exact + # round; the labeled series is evicted on the same cutoff as the other + # per-round families. Best-effort — never blocks scoring. + set_baseline_loss(round_obj.round_id, baseline_loss) foreground_set = set(round_obj.foreground_uids) completed: list[MinerEvalJob] = completed_out if completed_out is not None else []