diff --git a/connito/shared/config.py b/connito/shared/config.py index 6b34a5a..a4f00f7 100644 --- a/connito/shared/config.py +++ b/connito/shared/config.py @@ -362,6 +362,14 @@ class ValidatorCheckpointCfg(CheckpointCfg): cleanup_stale_temporary_checkpoints: bool = True miner_submission_max_age_cycles: PositiveFloat = 1.5 miner_submission_archive_max_files: PositiveInt = 500 + # Maximum number of UIDs that may sit in `Round.downloaded_pool` waiting + # for bg-eval to pick them up before bg-download stops fetching new + # checkpoints. Without this cap, bg-download will happily pull every + # miner's shard onto disk even when bg-eval is many minutes behind, which + # wastes HF bandwidth and (more importantly) inflates the on-disk backlog + # the cycle-tail prune has to tear down. Re-checked every poll so the cap + # self-clears once eval drains the queue. + download_pending_eval_cap: PositiveInt = 10 class DhtCfg(BaseConfig): diff --git a/connito/validator/background_download_worker.py b/connito/validator/background_download_worker.py index 70130d8..944eefd 100644 --- a/connito/validator/background_download_worker.py +++ b/connito/validator/background_download_worker.py @@ -39,15 +39,6 @@ logger = structlog.get_logger(__name__) -# Maximum number of UIDs that may sit in `Round.downloaded_pool` waiting -# for bg-eval to pick them up before bg-download stops fetching new -# checkpoints. Without this cap, bg-download will happily pull every -# miner's shard onto disk even when bg-eval is many minutes behind, which -# wastes HF bandwidth and (more importantly) inflates the on-disk backlog -# the cycle-tail prune has to tear down. Re-checked every poll so the cap -# self-clears once eval drains the queue. -DOWNLOAD_PENDING_EVAL_CAP = 10 - class BackgroundDownloadWorker(threading.Thread): def __init__( @@ -130,11 +121,12 @@ async def _loop(self) -> None: continue # Backpressure on bg-eval: stop pulling more checkpoints - # while bg-eval already has DOWNLOAD_PENDING_EVAL_CAP+ UIDs + # while bg-eval already has download_pending_eval_cap+ UIDs # queued. Counted under Round's lock so a concurrent # publish/pop can't skew the read. + pending_eval_cap = self.config.ckpt.download_pending_eval_cap pending_eval = round_obj.downloaded_pending_eval_count() - if pending_eval > DOWNLOAD_PENDING_EVAL_CAP: + if pending_eval > pending_eval_cap: # Log once on the rising edge into the cap; stay quiet # until a successful download resets idle_ticks (same # pattern as the "no pending targets" branch below). @@ -142,7 +134,7 @@ async def _loop(self) -> None: logger.info( "bg-download: pausing — eval backlog above cap", pending_eval=pending_eval, - cap=DOWNLOAD_PENDING_EVAL_CAP, + cap=pending_eval_cap, round_id=getattr(round_obj, "round_id", None), ) idle_ticks += 1