From 747c8e418590cae39653b53023c703e7e5fd3701 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 16 Jul 2026 07:07:09 -0500 Subject: [PATCH 1/5] chain_plugin: add is_synced(); gate operator-daemon startup on chain sync Every test cluster's boot window logged error-level lines from batch_operator (3060002/3060003): plugin_startup eagerly read sysio.chains before a cold-booting daemon node's local replay reached the deploy blocks. The underwriter already solved this privately with an accepted_block-signal gate plus an inner executor re-post. Promote the predicate - and only the predicate - to chain_plugin: - chain_plugin::is_synced(): LIB-recency predicate (fork_db root block time within chain_apis::default_sync_recency_ms of wall-clock now), computed on demand, never cached. The pure building blocks live in sync_gate.hpp (moved from the underwriter's sync_detail.hpp) with unit tests in test_sync_gate.cpp. - No new signal: the predicate is a level, and the wake-up a gated consumer needs already exists as the channels::irreversible_block appbase channel - a LIB advance is the only event that can turn the predicate true, and channel deliveries are posted to the application executor, so callbacks run on the main thread after the triggering block fully commits (the post-commit contract the underwriter previously re-implemented with its inner post). - underwriter_plugin: the gate subscription moves from the raw controller accepted_block signal to the channel; the private predicate, sync_recency_ms, and the inner re-post are deleted. - batch_operator_plugin: the startup body (outpost discovery, cron_service sizing/creation, epoch_tick, relay jobs) moves verbatim into impl::run_deferred_startup, gated the same way - the boot-window read errors disappear and cron sizing sees the real outpost count instead of always hitting the MIN_CRON_THREADS floor. - Single gated path, no already-synced fast path: operator daemons boot with genesis-stale LIB in every deployment topology (they are never co-hosted with a producer); a node that somehow is synced at startup is released by the next LIB advance. - Uniform fail-fast: a terminal deferred-startup failure logs and quits the node in both daemons - an operator daemon that is not performing its duties must be supervisor-visible, not hidden behind a running process. The underwriter's bounded preflight retry grace still absorbs transient failures; the escape_policy split (abort vs contain, previously selected implicitly by restart timing) is deleted, with underwriter_detail::is_terminal_failure classifying the terminal states in one place. - Both daemons warn when read-mode != irreversible - the gate and their local table reads serve the irreversible state. --- .../src/batch_operator_plugin.cpp | 178 +++++++++++---- .../sysio/chain_plugin/chain_plugin.hpp | 19 ++ .../include/sysio/chain_plugin/sync_gate.hpp | 81 +++++++ plugins/chain_plugin/src/chain_plugin.cpp | 6 + plugins/chain_plugin/test/CMakeLists.txt | 2 + plugins/chain_plugin/test/test_sync_gate.cpp | 96 +++++++++ .../sysio/underwriter_plugin/sync_detail.hpp | 70 +++--- .../underwriter_plugin/underwriter_plugin.hpp | 12 +- .../src/underwriter_plugin.cpp | 204 ++++++++---------- .../test/test_underwriter_sync.cpp | 71 ++---- 10 files changed, 491 insertions(+), 248 deletions(-) create mode 100644 plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp create mode 100644 plugins/chain_plugin/test/test_sync_gate.cpp diff --git a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp index 1575327b2f..c4173e7246 100644 --- a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp +++ b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -187,6 +189,13 @@ struct batch_operator_plugin::impl { std::map scheduled_opp_jobs; std::atomic shutting_down{false}; + /// Sync gate: `channels::irreversible_block` subscription that arms + /// {@link run_deferred_startup_or_quit} once `chain_plugin::is_synced()` + /// holds — a LIB advance is the only event that can turn the predicate + /// true (see `sysio/chain_plugin/sync_gate.hpp`). Unsubscribed after + /// arming; released on shutdown. + chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; + // ----------------------------------------------------------------------- // Chain-agnostic orchestration layer // ----------------------------------------------------------------------- @@ -812,6 +821,89 @@ struct batch_operator_plugin::impl { elog("batch_operator: push {}::{} timed out", contract, action_name); } } + + // ----------------------------------------------------------------------- + // Sync-gated startup + // ----------------------------------------------------------------------- + + /// The startup body deferred behind the chain_plugin sync gate: outpost + /// discovery → private cron_service creation (sized from the discovered + /// outposts) → epoch_tick scheduling → per-outpost relay jobs. Runs on the + /// main thread from {@link run_deferred_startup_or_quit} once the node is + /// synced. Deferral exists because `refresh_outposts` reads `sysio.chains` + /// LOCALLY: on a cold-booting operator node still replaying toward the + /// deploy blocks the read throws Account/Contract Query Exceptions + /// (3060002/3060003) — spurious boot-window errors the gate removes. + void run_deferred_startup() { + if (shutting_down) { + return; + } + + // Discover outposts before the private cron_service starts. Later refresh + // ticks add/remove per-outpost cron jobs as the active chain set changes. + try { + refresh_outposts(); + } catch (const fc::exception& e) { + wlog("batch_operator_plugin: initial outpost discovery failed: {}. " + "Starting with 0 per-outpost jobs; refresh ticks will retry after " + "the chain has caught up.", e.to_string()); + } + + // Size the pool for startup outposts. Later dynamic outposts are added to + // the same queued cron service; the minimum keeps epoch_tick viable even + // when no outposts are known yet. + const std::size_t outpost_count = opp_jobs.size(); + const std::size_t required_threads = outpost_count * OPP_CRON_JOBS_PER_OUTPOST + EPOCH_TICK_CRON_JOBS; + const std::size_t thread_count = std::max(required_threads, MIN_CRON_THREADS); + + sysio::services::cron_service::options svc_opts; + svc_opts.name = "batch_operator"; + svc_opts.num_threads = thread_count; + svc_opts.autostart = true; + + cron_svc = sysio::services::cron_service::create(svc_opts); + ilog("batch_operator_plugin: cron_service started with {} thread(s) ({} outpost(s) discovered)", + thread_count, outpost_count); + + const auto poll_ms = epoch_poll_ms; + + // epoch_tick — refresh epoch state + election. Keeps `current_epoch` + // and `within_epoch_window` accurate for every per-outpost job. + { + sysio::services::cron_service::job_schedule sched; + sched.milliseconds = {sysio::services::cron_service::job_schedule::step_value{poll_ms}}; + sysio::services::cron_service::job_metadata_t meta; + meta.label = "batch_operator_epoch_tick"; + meta.one_at_a_time = true; + auto id = cron_svc->add(sched, + [this]() { poll_epoch_state(); }, + meta); + cron_job_ids.push_back(id); + ilog("batch_operator_plugin: scheduled {} (id={}, every {}ms)", meta.label, id, poll_ms); + } + + schedule_opp_jobs(); + } + + /// {@link run_deferred_startup} plus the uniform fail-fast policy: the + /// sync-gate callback is a posted channel delivery, so an escaping + /// exception would unwind the application executor mid-task with no + /// diagnosable trace of WHAT failed. Contain it long enough to log, then + /// shut the node down — an operator daemon whose relay never started must + /// be supervisor-visible (it has liveness/slashing consequences), not + /// hidden behind a running process. Expected transient failures + /// (outpost discovery against a not-yet-deployed registry) are already + /// absorbed inside {@link run_deferred_startup} and retried by the refresh + /// ticks; what reaches here is structural (cron_service creation or job + /// scheduling failed). + void run_deferred_startup_or_quit() { + try { + run_deferred_startup(); + return; + } FC_LOG_AND_DROP("batch_operator_plugin: deferred startup failed unexpectedly:"); + elog("batch_operator_plugin: deferred startup failed terminally — shutting down node (fail-fast)"); + app().quit(); + } }; // --------------------------------------------------------------------------- @@ -882,54 +974,54 @@ void batch_operator_plugin::plugin_startup() { ilog("batch_operator_plugin: starting for account {}", _impl->operator_account.to_string()); - // Discover outposts before the private cron_service starts. Later refresh - // ticks add/remove per-outpost cron jobs as the active chain set changes. - try { - _impl->refresh_outposts(); - } catch (const fc::exception& e) { - wlog("batch_operator_plugin: initial outpost discovery failed: {}. " - "Starting with 0 per-outpost jobs; refresh ticks will retry after " - "the chain has caught up.", e.to_string()); + // The startup body's outpost discovery reads `sysio.chains` LOCALLY. On a + // cold-booting operator node those reads see mid-sync (possibly genesis) + // state and throw spuriously, so the whole body (discovery → cron_service → + // epoch_tick → relay jobs) is DEFERRED until the node is synced — + // `chain_plugin::is_synced()`: the LAST IRREVERSIBLE block's time within + // `chain_apis::default_sync_recency_ms` of now (the state the reads + // actually serve under read-mode = irreversible). The wake-up is the + // existing `irreversible_block` channel: a LIB advance is the only event + // that can turn the predicate true, and channel deliveries are posted to + // the application executor — main thread, AFTER the triggering block fully + // commits — so the callback may run the startup body directly. There is + // deliberately no already-synced fast path: operator daemons boot with + // genesis-stale LIB in every deployment topology (they are never co-hosted + // with a producer), and a node that somehow is synced at startup is + // released by the next LIB advance. + auto& chain = _impl->chain_plug->chain(); + if (chain.get_read_mode() != chain::db_read_mode::IRREVERSIBLE) { + wlog("batch_operator_plugin: node runs read-mode = {}, but operator daemons are designed " + "for read-mode = irreversible — the sync gate and every local table read serve " + "the irreversible state", magic_enum::enum_name(chain.get_read_mode())); } - - // Size the pool for startup outposts. Later dynamic outposts are added to - // the same queued cron service; the minimum keeps epoch_tick viable even - // when no outposts are known yet. - const std::size_t outpost_count = _impl->opp_jobs.size(); - const std::size_t required_threads = outpost_count * OPP_CRON_JOBS_PER_OUTPOST + EPOCH_TICK_CRON_JOBS; - const std::size_t thread_count = std::max(required_threads, MIN_CRON_THREADS); - - sysio::services::cron_service::options svc_opts; - svc_opts.name = "batch_operator"; - svc_opts.num_threads = thread_count; - svc_opts.autostart = true; - - _impl->cron_svc = sysio::services::cron_service::create(svc_opts); - ilog("batch_operator_plugin: cron_service started with {} thread(s) ({} outpost(s) discovered)", - thread_count, outpost_count); - - const auto poll_ms = _impl->epoch_poll_ms; - - // epoch_tick — refresh epoch state + election. Keeps `current_epoch` - // and `within_epoch_window` accurate for every per-outpost job. - { - sysio::services::cron_service::job_schedule sched; - sched.milliseconds = {sysio::services::cron_service::job_schedule::step_value{poll_ms}}; - sysio::services::cron_service::job_metadata_t meta; - meta.label = "batch_operator_epoch_tick"; - meta.one_at_a_time = true; - auto id = _impl->cron_svc->add(sched, - [impl = _impl.get()]() { impl->poll_epoch_state(); }, - meta); - _impl->cron_job_ids.push_back(id); - ilog("batch_operator_plugin: scheduled {} (id={}, every {}ms)", meta.label, id, poll_ms); - } - - _impl->schedule_opp_jobs(); + ilog("batch_operator_plugin: waiting for chain sync before outpost discovery " + "(head {} is {}s behind now; irreversible state is {}s behind)", + chain.head().block_num(), + (fc::time_point::now() - chain.head().block_time()).to_seconds(), + chain.fork_db_has_root() + ? (fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds() + : -1); + _impl->sync_gate_subscription = + app().get_channel().subscribe( + [impl = _impl.get()](const chain::block_signal_params&) { + if (impl->shutting_down || !impl->chain_plug->is_synced()) { + return; + } + // One-shot consumption: unsubscribe (safe from within the slot) and + // run the startup body directly — channel deliveries are posted to + // the application executor, so this already runs on the main thread + // AFTER the triggering block committed (mid block-application, + // table reads would observe an incomplete view). + impl->sync_gate_subscription.unsubscribe(); + ilog("batch_operator_plugin: chain synced — starting deferred startup"); + impl->run_deferred_startup_or_quit(); + }); } void batch_operator_plugin::plugin_shutdown() { _impl->shutting_down = true; + _impl->sync_gate_subscription.unsubscribe(); if (_impl->cron_svc) { _impl->cron_svc->cancel_all(); _impl->cron_svc->stop(); diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 1640c7e149..2b433900f8 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -695,6 +695,25 @@ class chain_plugin : public plugin { // Only call this after plugin_initialize()! const controller& chain() const; + /// The "node is synced" predicate: true when the LAST IRREVERSIBLE block's time is + /// within `chain_apis::default_sync_recency_ms` of wall-clock now (see + /// `sysio/chain_plugin/sync_gate.hpp`). LIB recency, not head recency, deliberately — + /// the operator-daemon plugins this gates run read-mode = irreversible, so the + /// irreversible state is what their table reads actually serve. False while no + /// irreversible root exists (fresh boot, mid-replay, snapshot catch-up). Computed on + /// demand, never cached, so a stalled chain reads as NOT synced as soon as its LIB + /// leaves the window. + /// + /// Consumers deferring startup until the node is synced subscribe to the existing + /// `chain::plugin_interface::channels::irreversible_block` channel (a LIB advance is + /// the only event that can turn this predicate true), re-check per delivery, and + /// unsubscribe once it holds. Channel deliveries are posted to the application + /// executor, so they run on the main thread AFTER the triggering block fully + /// commits — never mid block-application — and may read chain state directly. + /// + /// Only call this after plugin_initialize()! + bool is_synced() const; + chain::chain_id_type get_chain_id() const; fc::microseconds get_abi_serializer_max_time() const; bool api_accept_transactions() const; diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp new file mode 100644 index 0000000000..ee6bd73d89 --- /dev/null +++ b/plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp @@ -0,0 +1,81 @@ +#pragma once +/** + * @file sync_gate.hpp + * @brief Pure building blocks of the chain_plugin sync gate — the predicate behind + * `chain_plugin::is_synced()` — lifted out of the `.cpp`-private impl so they are + * unit-testable without standing up a chain. + * + * Plugins whose startup validates on-chain state via LOCAL table reads (the operator + * daemons: batch_operator_plugin, underwriter_plugin) must not start until the node is + * synced. On a cold-booting operator node those reads see mid-sync (possibly genesis) + * state and fail spuriously. "Synced" here means the LAST IRREVERSIBLE block's time is + * within a small window of wall-clock now — the irreversible state is what those + * plugins' table reads actually serve (operator daemons run read-mode = irreversible). + * + * There is deliberately no push mechanism here: the predicate is a LEVEL, evaluated on + * demand (never cached — a cached flag updated on block events would go stale-TRUE the + * moment the chain stalls), and the wake-up a gated consumer needs already exists as + * the `chain::plugin_interface::channels::irreversible_block` channel — a LIB advance + * is the only event that can turn the predicate true, and channel deliveries are posted + * to the application executor (main thread, AFTER the triggering block fully commits — + * never mid block-application, where table reads observe an incomplete view). Consumers + * subscribe, re-check `is_synced()` per delivery, and unsubscribe once it holds. + */ + +#include + +#include + +namespace sysio::chain_apis { + + /// Behind-now gap (ms) under which the node counts as synced. Measured against the + /// LAST IRREVERSIBLE block's time — the state operator-daemon table reads actually + /// serve (read-mode = irreversible); see {@link lib_time_is_recent}. Must exceed + /// steady-state finality lag (a few blocks) plus scheduling slack, and stay well + /// under one epoch so gated consumers never start against stale state. Not an + /// operator tunable. + inline constexpr uint32_t default_sync_recency_ms = 5000; + + /** + * @brief True when a block time is recent enough to treat the node as synced. + * + * While a cold-booting node syncs blocks the tested time trails `now` by the + * catch-up gap; once caught up, it tracks `now` to within finality-lag jitter. The + * window must exceed that jitter (plus scheduling slack) but stay well under one + * epoch so gated consumers never start against stale state. + * + * @param block_time The block timestamp to test — the caller's sync criterion + * (the sync gate feeds the LAST IRREVERSIBLE block's time, + * since that is the state operator-daemon reads serve). + * @param now Wall-clock now. + * @param recency_window Maximum behind-now gap still considered synced. + * @return True when `block_time >= now - recency_window`. + */ + inline bool block_time_is_recent(fc::time_point block_time, fc::time_point now, + fc::microseconds recency_window) { + return block_time >= now - recency_window; + } + + /** + * @brief The "node is synced" predicate: the LAST IRREVERSIBLE block's time within + * `recency_window` of `now`. + * + * LIB recency, not head recency, deliberately: operator daemons run read-mode = + * irreversible, so their table reads serve the IRREVERSIBLE state. Head-time + * recency armed the underwriter's original gate while the local LIB still trailed + * the rows its preflight needed (observed live: a registration in block N was + * readable only 50ms after a preflight read at LIB N−3). False while no + * irreversible root exists. + * + * @param chain The controller to read the irreversible root from. + * @param now Wall-clock now. + * @param recency_window Maximum behind-now gap still considered synced. + * @return True when an irreversible root exists and its block time is recent. + */ + inline bool lib_time_is_recent(const chain::controller& chain, fc::time_point now, + fc::microseconds recency_window) { + return chain.fork_db_has_root() && + block_time_is_recent(chain.fork_db_root().block_time(), now, recency_window); + } + +} // namespace sysio::chain_apis diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index 080b2cfcbb..b3a338f7dd 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -1499,6 +1500,11 @@ void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, controller& chain_plugin::chain() { return *my->chain; } const controller& chain_plugin::chain() const { return *my->chain; } +bool chain_plugin::is_synced() const { + return chain_apis::lib_time_is_recent(chain(), fc::time_point::now(), + fc::milliseconds(chain_apis::default_sync_recency_ms)); +} + chain::chain_id_type chain_plugin::get_chain_id()const { return my->chain->get_chain_id(); } diff --git a/plugins/chain_plugin/test/CMakeLists.txt b/plugins/chain_plugin/test/CMakeLists.txt index 9e30f3f891..dfc2bdc197 100644 --- a/plugins/chain_plugin/test/CMakeLists.txt +++ b/plugins/chain_plugin/test/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable( test_chain_plugin test_account_query_db.cpp + test_sync_gate.cpp test_trx_retry_db.cpp test_trx_finality_status_processing.cpp plugin_config_test.cpp @@ -9,6 +10,7 @@ target_link_libraries( test_chain_plugin chain_plugin sysio_testing sysio_chain_ set(CHAIN_PLUGIN_TEST_CASES chain_plugin_default_tests account_query_db_tests + sync_gate_tests trx_finality_status_processing_test trx_retry_db_test ) diff --git a/plugins/chain_plugin/test/test_sync_gate.cpp b/plugins/chain_plugin/test/test_sync_gate.cpp new file mode 100644 index 0000000000..68a77851dc --- /dev/null +++ b/plugins/chain_plugin/test/test_sync_gate.cpp @@ -0,0 +1,96 @@ +/** + * @file test_sync_gate.cpp + * @brief Unit tests for the chain_plugin sync gate (`sysio/chain_plugin/sync_gate.hpp`) + * — the LIB-recency predicate behind `chain_plugin::is_synced()`. + * + * The pure-predicate cases migrated verbatim from the underwriter plugin's private + * gate (`test_underwriter_sync.cpp`) when the gate was promoted to chain_plugin. + */ +#include + +#include +#include + +using namespace sysio::testing; +using sysio::chain_apis::block_time_is_recent; +using sysio::chain_apis::lib_time_is_recent; + +BOOST_AUTO_TEST_SUITE(sync_gate_tests) + +/// The reference "now" every pure case offsets from — an arbitrary fixed instant so +/// the tests are deterministic (no wall clock). +static const fc::time_point reference_now = + fc::time_point{} + fc::days(365 * 50); + +/// The default gate window the plugin runs with. +static const fc::microseconds default_window = + fc::milliseconds(sysio::chain_apis::default_sync_recency_ms); + +// ── block_time_is_recent: the pure recency predicate ── + +BOOST_AUTO_TEST_CASE(head_at_now_is_synced) { + BOOST_TEST(block_time_is_recent(reference_now, reference_now, default_window)); +} + +BOOST_AUTO_TEST_CASE(head_slightly_behind_within_window_is_synced) { + // One block interval behind — the steady-state of a caught-up node. + BOOST_TEST(block_time_is_recent(reference_now - fc::milliseconds(500), + reference_now, default_window)); +} + +BOOST_AUTO_TEST_CASE(head_exactly_at_window_boundary_is_synced) { + BOOST_TEST(block_time_is_recent(reference_now - default_window, + reference_now, default_window)); +} + +BOOST_AUTO_TEST_CASE(head_just_past_window_is_not_synced) { + BOOST_TEST(!block_time_is_recent(reference_now - default_window - fc::microseconds(1), + reference_now, default_window)); +} + +BOOST_AUTO_TEST_CASE(cold_boot_replay_gap_is_not_synced) { + // A node mid-replay: head hours behind wall clock — the exact state whose + // genesis-era table reads used to false-fail operator-daemon startups. + BOOST_TEST(!block_time_is_recent(reference_now - fc::hours(2), + reference_now, default_window)); +} + +BOOST_AUTO_TEST_CASE(head_ahead_of_now_is_synced) { + // Clock skew / head stamped marginally ahead of the observer's clock must + // never read as "behind". + BOOST_TEST(block_time_is_recent(reference_now + fc::seconds(1), + reference_now, default_window)); +} + +BOOST_AUTO_TEST_CASE(zero_window_requires_head_at_or_past_now) { + BOOST_TEST(block_time_is_recent(reference_now, reference_now, fc::microseconds(0))); + BOOST_TEST(!block_time_is_recent(reference_now - fc::microseconds(1), + reference_now, fc::microseconds(0))); +} + +// ── lib_time_is_recent: the controller-backed predicate `is_synced()` delegates to ── + +BOOST_AUTO_TEST_CASE(stale_chain_is_not_synced) { try { + // The tester genesis timestamp is a fixed instant years in the past, so a + // freshly-produced chain's irreversible root time trails wall-clock now by + // that whole gap — the cold-boot replay state the gate exists to detect. + tester chain; + chain.produce_blocks(4); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + BOOST_TEST(!lib_time_is_recent(*chain.control, fc::time_point::now(), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE(fresh_chain_is_synced) { try { + // Jump block time from the fixed genesis instant to wall-clock now, then + // produce a few more blocks so the near-now block becomes irreversible. The + // subsequent block times land at/ahead of now (ahead counts as recent — + // clock-skew rule above), so the LIB-recency predicate must hold. + tester chain; + chain.produce_block(); + chain.produce_block(fc::time_point::now() - chain.control->head().block_time()); + chain.produce_blocks(6); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + BOOST_TEST(lib_time_is_recent(*chain.control, fc::time_point::now(), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp index e1a837bcf2..0a01bfddb5 100644 --- a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp +++ b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp @@ -1,17 +1,20 @@ #pragma once /** * @file sync_detail.hpp - * @brief Pure sync-gate predicate for the underwriter plugin, lifted out of the - * `.cpp`-private impl so it is unit-testable without standing up a chain. + * @brief Pure gate-lifecycle machinery for the underwriter plugin — the deferred- + * startup states and the `/v1/underwriter/*` gate payloads — lifted out of + * the `.cpp`-private impl so they are unit-testable without standing up a + * chain. * * The underwriter's startup preflight validates depot-side state (opreg * registration, chain registry, authex links) via LOCAL table reads. On a * cold-booting operator node those reads see mid-sync (possibly genesis) * state and fail spuriously, so the plugin must not begin underwriting until - * the node is synced. "Synced" here means the LAST IRREVERSIBLE block's time - * is within a small window of wall-clock now — the irreversible state is what - * the plugin's table reads actually serve (operator daemons run read-mode = - * irreversible). + * the node is synced. The sync predicate itself is the first-class chain_plugin + * API — `chain_plugin::is_synced()`, backed by + * `sysio/chain_plugin/sync_gate.hpp` and woken by the `irreversible_block` + * channel — shared by every operator-daemon plugin; this header carries only + * the underwriter's OWN gate lifecycle surface. */ #include @@ -23,26 +26,6 @@ namespace sysio::underwriter_detail { -/** - * @brief True when a block time is recent enough to treat the node as synced. - * - * While a cold-booting node syncs blocks the tested time trails `now` by the - * catch-up gap; once caught up, it tracks `now` to within finality-lag - * jitter. The window must exceed that jitter (plus scheduling slack) but stay - * well under one epoch so underwriting never starts against stale state. - * - * @param block_time The block timestamp to test — the caller's sync - * criterion (the underwriter feeds the LAST IRREVERSIBLE - * block's time, since that is the state its reads serve). - * @param now Wall-clock now. - * @param recency_window Maximum behind-now gap still considered synced. - * @return True when `block_time >= now - recency_window`. - */ -inline bool block_time_is_recent(fc::time_point block_time, fc::time_point now, - fc::microseconds recency_window) { - return block_time >= now - recency_window; -} - /// JSON field keys shared by the gate payloads below, the post-startup /// response builders in `underwriter_plugin.cpp`, and their tests — one /// spelling authority instead of per-translation-unit string literals. @@ -73,13 +56,30 @@ enum class startup_state : uint8_t { wiring_failed, ///< Preflight passed but outpost client wiring threw (terminal). startup_failed, ///< The deferred startup body threw past the specific ///< failure paths above (terminal). Exists so an escaping - ///< exception is contained as a diagnosable gate state - ///< instead of unwinding the app executor — which would - ///< tear down the whole node — with the gate still - ///< claiming `waiting_for_sync`. + ///< exception is contained as a diagnosable gate state — + ///< stored and logged BEFORE the fail-fast shutdown — instead + ///< of unwinding the app executor mid-task with the gate + ///< still claiming `waiting_for_sync`. active ///< Deferred startup completed — the scan cron is scheduled. }; +/** + * @brief True for the terminal deferred-startup FAILURE states — the states the + * uniform fail-fast policy shuts the node down on (see + * `underwriter_plugin.cpp`'s `quit_if_startup_failed_terminally`). + * + * `active` is terminal success, not failure; `preflight_retrying` is transient by + * definition (bounded by the retry grace). Kept beside the enum so a new state + * cannot be added without deciding its fail-fast classification here. + * + * @param s The deferred-startup lifecycle state to classify. + * @return True when `s` is a terminal failure state. + */ +inline constexpr bool is_terminal_failure(startup_state s) { + return s == startup_state::preflight_failed || s == startup_state::wiring_failed || + s == startup_state::startup_failed; +} + /// The wire `status` spelling of {@link startup_state::active} — shared by the /// post-startup response builders so the value has one authority beside the /// enum whose member spellings define the wire format. @@ -91,15 +91,15 @@ static_assert(active_status == "active"); /// `detail` carried by the {@link startup_state::preflight_failed} gate payload. inline constexpr std::string_view preflight_failed_detail = "startup preflight failed after sync — depot-side state for this underwriter " - "is incomplete; underwriting is disabled (see node log)"; + "is incomplete; the node is shutting down (fail-fast; see node log)"; /// `detail` carried by the {@link startup_state::wiring_failed} gate payload. inline constexpr std::string_view wiring_failed_detail = - "outpost client wiring failed after preflight; underwriting is disabled " - "(see node log)"; + "outpost client wiring failed after preflight; the node is shutting down " + "(fail-fast; see node log)"; /// `detail` carried by the {@link startup_state::startup_failed} gate payload. inline constexpr std::string_view startup_failed_detail = - "deferred startup failed unexpectedly; underwriting is disabled " - "(see node log)"; + "deferred startup failed unexpectedly; the node is shutting down " + "(fail-fast; see node log)"; /** * @brief The JSON body a `/v1/underwriter/...` endpoint answers with for `state`. diff --git a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp index abc0d7f140..b9303beac1 100644 --- a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp +++ b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp @@ -23,14 +23,10 @@ namespace sysio { constexpr uint32_t scan_interval_ms = 5000; constexpr uint32_t action_timeout_ms = 15000; constexpr bool enabled = false; - /// Behind-now gap (ms) under which the node counts as synced. Measured - /// against the LAST IRREVERSIBLE block's time — the state the plugin's - /// table reads actually serve (operator daemons run read-mode = - /// irreversible). Gates the deferred startup (preflight → cron) on - /// cold-booting operator nodes; see `sync_detail.hpp::block_time_is_recent`. - /// Must exceed steady-state finality lag (a few blocks) and stay well - /// under one epoch. - constexpr uint32_t sync_recency_ms = 5000; + // The sync-recency window moved to `chain_apis::default_sync_recency_ms` + // (sysio/chain_plugin/sync_gate.hpp) — the sync predicate is the + // first-class chain_plugin API (`is_synced()`), shared by every + // operator-daemon plugin. /// How long after the sync gate arms a failing preflight keeps /// RETRYING before the failure is terminal. Rows the harness confirmed /// final on the producer can land in the LOCAL irreversible state a diff --git a/plugins/underwriter_plugin/src/underwriter_plugin.cpp b/plugins/underwriter_plugin/src/underwriter_plugin.cpp index e1bbc3d07a..b38d38126c 100644 --- a/plugins/underwriter_plugin/src/underwriter_plugin.cpp +++ b/plugins/underwriter_plugin/src/underwriter_plugin.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -245,11 +246,12 @@ struct underwriter_plugin::impl { cron_service::job_id_t scan_job_id = 0; std::atomic shutting_down{false}; - /// Sync gate: `accepted_block` subscription that arms {@link run_deferred_startup} - /// once the local irreversible state is current (see - /// `sync_detail.hpp::block_time_is_recent`). Disconnected after arming; reset on + /// Sync gate: `channels::irreversible_block` subscription that arms + /// {@link run_deferred_startup} once `chain_plugin::is_synced()` holds — a LIB + /// advance is the only event that can turn the predicate true (see + /// `sysio/chain_plugin/sync_gate.hpp`). Unsubscribed after arming; released on /// shutdown. - std::optional sync_gate_connection; + chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; /// When the sync gate armed — bounds the preflight retry grace window. fc::time_point startup_armed_at; /// Bounded-grace preflight retry timer (main io_context; main-thread wait). @@ -749,48 +751,19 @@ struct underwriter_plugin::impl { // Sync-gated startup // ----------------------------------------------------------------------- - /// The "node is synced" predicate: the LAST IRREVERSIBLE block's time - /// within `underwriter_defaults::sync_recency_ms` of wall-clock now. - /// The plugin's table reads serve the IRREVERSIBLE state (operator - /// daemons run read-mode = irreversible), so the gate must measure LIB - /// recency — head-time recency armed the gate while the local LIB still - /// trailed the rows the preflight needs (observed live: a registration - /// in block N was readable only 50ms after a preflight read at LIB N−3). - /// While a cold-booting operator node syncs blocks, LIB trails `now` by - /// the catch-up gap and every local table read sees stale (possibly - /// genesis) state — preflight must not run until this is true. - bool chain_is_synced() const { - auto& chain = chain_plug->chain(); - if (!chain.fork_db_has_root()) { - return false; - } - return underwriter_detail::block_time_is_recent( - chain.fork_db_root().block_time(), fc::time_point::now(), - fc::milliseconds(underwriter_defaults::sync_recency_ms)); - } - - /// What an exception escaping the deferred startup body does. The - /// synchronous `plugin_startup` path must ABORT node startup — fail-fast, - /// supervisor-visible; booting without underwriting would hide a broken - /// deployment behind one log line. Tasks posted after `exec()` is live - /// must CONTAIN the escape as a terminal gate state instead: unwinding - /// out of a posted task makes appbase `quit()` and tear down a node that - /// may be serving other roles. - enum class escape_policy : uint8_t { abort_on_escape, contain_escapes }; - /// Arm the deferred startup: runs AT MOST once (re-entry is guarded by - /// {@link startup_attempted}), on the main thread (either directly from - /// `plugin_startup` when the node is already synced, or from the - /// `accepted_block` signal once it becomes synced). The body itself lives + /// {@link startup_attempted}), on the main thread from the sync-gate channel + /// callback once `chain_plugin::is_synced()` holds. The body itself lives /// in {@link attempt_deferred_startup}, which may retry the preflight - /// within a bounded grace. - void run_deferred_startup(escape_policy policy) { + /// within a bounded grace and shuts the node down on terminal failure + /// (fail-fast). + void run_deferred_startup() { startup_armed_at = fc::time_point::now(); - // Release the gate slot; the callback already disconnected it before - // posting, so this is an idempotent cleanup, not the load-bearing - // disconnect. - sync_gate_connection.reset(); - attempt_deferred_startup(policy); + // Release the gate subscription; the callback already unsubscribed + // before calling here, so this is an idempotent cleanup, not the + // load-bearing disconnect. + sync_gate_subscription.unsubscribe(); + attempt_deferred_startup(); } /// One attempt of the deferred startup body: preflight → outpost client @@ -801,44 +774,57 @@ struct underwriter_plugin::impl { /// (observed live: `regoperator` in block 405 readable 50ms after a /// preflight read at LIB 402), so the attempt re-schedules itself on /// `preflight_retry_interval_ms` until the grace expires. Past the grace, - /// a preflight failure on a synced chain remains a loud, terminal - /// bootstrap bug — no cron job, no dev escape hatch — exactly as before. - void attempt_deferred_startup(escape_policy policy) { + /// a preflight failure on a synced chain is a terminal bootstrap bug — + /// stored as a diagnosable gate state, logged, and answered with a + /// fail-fast node shutdown (see {@link quit_if_startup_failed_terminally}). + /// + /// Escapes are contained FIRST (unwinding out of a posted channel delivery + /// would skip the diagnosable gate-state store and the shutdown log) and + /// then routed through the same fail-fast shutdown. The expected + /// preflight/wiring failures inside the body set their own more specific + /// states; the containment catches what they don't (a throwing table + /// decode in the preflight or registry read, a non-fc exception from + /// client wiring, a cron add_job failure). FC_LOG_AND_DROP is the tree's + /// containment idiom (see scan_cycle below) and deliberately rethrows + /// boost::interprocess::bad_alloc — chainbase shared-memory exhaustion + /// stays immediately fatal. + void attempt_deferred_startup() { if (shutting_down) { return; } - if (policy == escape_policy::abort_on_escape) { - // Synchronous plugin_startup path: let an escape abort node startup - // (nodeop exits nonzero) exactly as before the sync gate existed. + bool completed = false; + try { attempt_deferred_startup_body(); - } else { - // Posted contexts (the sync-gate callback's task and the preflight - // retry timer): contain every escape as a terminal, diagnosable gate - // state — unwinding out of a posted task makes appbase quit() and - // tear down the whole node, with the gate still claiming - // waiting_for_sync. The expected preflight/wiring failures inside - // the body set their own more specific states; this catches what - // they don't (a throwing table decode in the preflight or registry - // read, a non-fc exception from client wiring, a cron add_job - // failure). FC_LOG_AND_DROP is the tree's containment idiom (see - // scan_cycle below) and deliberately rethrows - // boost::interprocess::bad_alloc — chainbase shared-memory - // exhaustion must stay fatal rather than linger as a gate state. - bool completed = false; - try { - attempt_deferred_startup_body(); - completed = true; - } FC_LOG_AND_DROP("underwriter_plugin: deferred startup failed unexpectedly:"); - if (!completed) { - gate_state = underwriter_detail::startup_state::startup_failed; - } + completed = true; + } FC_LOG_AND_DROP("underwriter_plugin: deferred startup failed unexpectedly:"); + if (!completed) { + gate_state = underwriter_detail::startup_state::startup_failed; } // Tripwire for the {@link startup_attempted} derivation: every attempt // that returns normally must have left `waiting_for_sync` — a future // early return in the body before its first gate_state store would - // otherwise strand the node in undiagnosable limbo (gate connection + // otherwise strand the node in undiagnosable limbo (gate subscription // already released, nothing left to re-arm). assert(startup_attempted()); + quit_if_startup_failed_terminally(); + } + + /// The uniform fail-fast policy: a terminal deferred-startup failure shuts + /// the node down instead of leaving a quiet, non-underwriting node behind a + /// running process — an operator daemon that is not performing its duties + /// must be supervisor-visible (it has liveness/slashing consequences), not + /// hidden behind one log line. The specific failure was already stored as a + /// gate state and logged by the attempt; this holds the shutdown decision in + /// ONE place so the policy cannot diverge between the first attempt and the + /// bounded preflight retries. + void quit_if_startup_failed_terminally() { + const auto state = gate_state.load(); + if (!underwriter_detail::is_terminal_failure(state)) { + return; + } + elog("underwriter_plugin: deferred startup failed terminally (state={}) — " + "shutting down node (fail-fast)", magic_enum::enum_name(state)); + app().quit(); } /// The deferred startup body proper — see {@link attempt_deferred_startup} @@ -966,7 +952,7 @@ struct underwriter_plugin::impl { gate_state.load() != underwriter_detail::startup_state::preflight_retrying) { return; } - attempt_deferred_startup(escape_policy::contain_escapes); + attempt_deferred_startup(); }); }); } @@ -2770,23 +2756,28 @@ void underwriter_plugin::plugin_startup() { // registry, authex links) via LOCAL table reads. On a cold-booting // operator node those reads see mid-sync (possibly genesis) state and // would fail spuriously, so the whole startup body (preflight → outpost - // client wiring → cron) is DEFERRED until the node is synced — the LAST - // IRREVERSIBLE block's time within `underwriter_defaults::sync_recency_ms` - // of now (the state the reads actually serve under read-mode = - // irreversible), observed via the controller's `accepted_block` signal. - // Post-arming, the preflight retries within a bounded grace (see - // attempt_deferred_startup) to absorb the LIB boundary race; a genuinely - // incomplete bootstrap still fails loudly once the grace expires — there - // remains no dev escape hatch. + // client wiring → cron) is DEFERRED until the node is synced — + // `chain_plugin::is_synced()`: the LAST IRREVERSIBLE block's time within + // `chain_apis::default_sync_recency_ms` of now (the state the reads + // actually serve under read-mode = irreversible). The wake-up is the + // existing `irreversible_block` channel: a LIB advance is the only event + // that can turn the predicate true, and channel deliveries are posted to + // the application executor — main thread, AFTER the triggering block + // fully commits — so the callback may run the startup body directly. + // There is deliberately no already-synced fast path: operator daemons + // boot with genesis-stale LIB in every deployment topology (they are + // never co-hosted with a producer), and a node that somehow is synced at + // startup is released by the next LIB advance. Post-arming, the preflight + // retries within a bounded grace (see attempt_deferred_startup) to absorb + // the LIB boundary race; a genuinely incomplete bootstrap still fails + // loudly once the grace expires and shuts the node down (fail-fast) — + // there remains no dev escape hatch. auto& chain = _impl->chain_plug->chain(); - if (_impl->chain_is_synced()) { - // Synchronous boot path: an escape aborts node startup (fail-fast, - // supervisor-visible) rather than booting with underwriting silently - // disabled. - _impl->run_deferred_startup(impl::escape_policy::abort_on_escape); - return; + if (chain.get_read_mode() != chain::db_read_mode::IRREVERSIBLE) { + wlog("underwriter_plugin: node runs read-mode = {}, but operator daemons are designed " + "for read-mode = irreversible — the sync gate and every local table read serve " + "the irreversible state", magic_enum::enum_name(chain.get_read_mode())); } - ilog("underwriter_plugin: waiting for chain sync before preflight " "(head {} is {}s behind now; irreversible state is {}s behind)", chain.head().block_num(), @@ -2794,35 +2785,30 @@ void underwriter_plugin::plugin_startup() { chain.fork_db_has_root() ? (fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds() : -1); - _impl->sync_gate_connection.emplace( - chain.accepted_block().connect([this](const chain::block_signal_params&) { - if (_impl->startup_attempted() || _impl->shutting_down) { - return; - } - if (!_impl->chain_is_synced()) { - return; - } - // Detected sync — disconnect the gate (safe mid-invocation) and POST - // the startup body to the app queue. It must NOT run inline here: - // the accepted_block signal fires mid block-application, where the - // chain API table reads see an incomplete view (observed live: a - // registered operator row read back EMPTY → spurious preflight - // failure). The posted task runs after the block commits, in the - // same main-thread context plugin_startup itself runs in. - _impl->sync_gate_connection->disconnect(); - ilog("underwriter_plugin: chain synced — scheduling deferred startup"); - app().executor().post(appbase::priority::medium, [this]() { - if (_impl->startup_attempted() || _impl->shutting_down) { + _impl->sync_gate_subscription = + app().get_channel().subscribe( + [this](const chain::block_signal_params&) { + if (_impl->startup_attempted() || _impl->shutting_down || + !_impl->chain_plug->is_synced()) { return; } - _impl->run_deferred_startup(impl::escape_policy::contain_escapes); + // One-shot consumption: unsubscribe (safe from within the slot) and + // run the startup body directly — channel deliveries are posted to + // the application executor, so this already runs on the main thread + // AFTER the triggering block committed (the exact context the old + // accepted_block gate had to re-post itself into; running mid + // block-application, table reads see an incomplete view — observed + // live: a registered operator row read back EMPTY → spurious + // preflight failure). + _impl->sync_gate_subscription.unsubscribe(); + ilog("underwriter_plugin: chain synced — starting deferred startup"); + _impl->run_deferred_startup(); }); - })); } void underwriter_plugin::plugin_shutdown() { _impl->shutting_down = true; - _impl->sync_gate_connection.reset(); + _impl->sync_gate_subscription.unsubscribe(); if (_impl->preflight_retry_timer) { _impl->preflight_retry_timer->cancel(); } diff --git a/plugins/underwriter_plugin/test/test_underwriter_sync.cpp b/plugins/underwriter_plugin/test/test_underwriter_sync.cpp index ae08632646..4c4a86e4ed 100644 --- a/plugins/underwriter_plugin/test/test_underwriter_sync.cpp +++ b/plugins/underwriter_plugin/test/test_underwriter_sync.cpp @@ -1,66 +1,31 @@ /** * @file test_underwriter_sync.cpp - * @brief Unit tests for the underwriter plugin's sync-gate predicate - * (`sysio::underwriter_detail::block_time_is_recent`) — the pure decision - * behind deferring preflight until the chain plugin reports the node - * synced. + * @brief Unit tests for the underwriter plugin's sync-gate lifecycle surface — + * the `/v1/underwriter/*` gate payloads served until the deferred startup + * completes, and the terminal-failure classification behind the fail-fast + * shutdown. The sync predicate itself is the first-class chain_plugin + * gate; its tests live in `plugins/chain_plugin/test/test_sync_gate.cpp`. */ #include #include #include -using sysio::underwriter_detail::block_time_is_recent; - BOOST_AUTO_TEST_SUITE(underwriter_sync_tests) -/// The reference "now" every case offsets from — an arbitrary fixed instant so -/// the tests are deterministic (no wall clock). -static const fc::time_point reference_now = - fc::time_point{} + fc::days(365 * 50); - -/// The default gate window the plugin runs with. -static const fc::microseconds default_window = - fc::milliseconds(sysio::underwriter_defaults::sync_recency_ms); - -BOOST_AUTO_TEST_CASE(head_at_now_is_synced) { - BOOST_TEST(block_time_is_recent(reference_now, reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_slightly_behind_within_window_is_synced) { - // One block interval behind — the steady-state of a caught-up node. - BOOST_TEST(block_time_is_recent(reference_now - fc::milliseconds(500), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_exactly_at_window_boundary_is_synced) { - BOOST_TEST(block_time_is_recent(reference_now - default_window, - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_just_past_window_is_not_synced) { - BOOST_TEST(!block_time_is_recent(reference_now - default_window - fc::microseconds(1), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(cold_boot_replay_gap_is_not_synced) { - // A node mid-replay: head hours behind wall clock — the exact state whose - // genesis-era table reads used to false-fail the preflight. - BOOST_TEST(!block_time_is_recent(reference_now - fc::hours(2), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_ahead_of_now_is_synced) { - // Clock skew / head stamped marginally ahead of the observer's clock must - // never read as "behind". - BOOST_TEST(block_time_is_recent(reference_now + fc::seconds(1), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(zero_window_requires_head_at_or_past_now) { - BOOST_TEST(block_time_is_recent(reference_now, reference_now, fc::microseconds(0))); - BOOST_TEST(!block_time_is_recent(reference_now - fc::microseconds(1), - reference_now, fc::microseconds(0))); +// ── is_terminal_failure: the fail-fast trigger behind the node shutdown ── + +BOOST_AUTO_TEST_CASE(terminal_failure_states_are_exactly_the_failed_states) { + using sysio::underwriter_detail::is_terminal_failure; + using sysio::underwriter_detail::startup_state; + // Non-failures: the pre-arm wait, the bounded transient retry, and success. + BOOST_TEST(!is_terminal_failure(startup_state::waiting_for_sync)); + BOOST_TEST(!is_terminal_failure(startup_state::preflight_retrying)); + BOOST_TEST(!is_terminal_failure(startup_state::active)); + // Failures: each one must shut the node down (fail-fast). + BOOST_TEST(is_terminal_failure(startup_state::preflight_failed)); + BOOST_TEST(is_terminal_failure(startup_state::wiring_failed)); + BOOST_TEST(is_terminal_failure(startup_state::startup_failed)); } // ── startup_gate_payload: the /v1/underwriter/* body served until the ── From 076dc3de5f85f60297d8e84fbeb86a6268331ffe Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 16 Jul 2026 07:36:15 -0500 Subject: [PATCH 2/5] chain: move is_synced() onto controller; require irreversible read-mode in operator daemons - controller::is_synced(now, recency_window) + wall-clock is_synced(): the LIB-recency predicate moves from chain_plugin (sync_gate.hpp, deleted) onto the controller, beside the fork_db accessors it reads - thread-safe like them. default_sync_recency_ms becomes a controller class constant; chain_plugin::is_synced() is deleted and consumers call chain().is_synced(). - Tests move to unittests/controller_sync_tests.cpp and now drive the real controller code path: the boundary cases feed is_synced(now, window) instants offset from the tester chain's actual last-irreversible block time, so every comparison is deterministic without a wall clock. - read-mode = irreversible is now REQUIRED by batch_operator_plugin and underwriter_plugin (FC_ASSERT at plugin_initialize when the plugin is enabled) instead of warned about: the sync gate measures LIB recency and every local table read the daemons perform serves the irreversible view; any other read mode would act on state that can still fork out. --- libraries/chain/controller.cpp | 8 ++ .../chain/include/sysio/chain/controller.hpp | 39 ++++++++ .../src/batch_operator_plugin.cpp | 27 ++--- .../sysio/chain_plugin/chain_plugin.hpp | 19 ---- .../include/sysio/chain_plugin/sync_gate.hpp | 81 --------------- plugins/chain_plugin/src/chain_plugin.cpp | 6 -- plugins/chain_plugin/test/CMakeLists.txt | 2 - plugins/chain_plugin/test/test_sync_gate.cpp | 96 ------------------ .../sysio/underwriter_plugin/sync_detail.hpp | 9 +- .../underwriter_plugin/underwriter_plugin.hpp | 5 +- .../src/underwriter_plugin.cpp | 29 +++--- unittests/controller_sync_tests.cpp | 98 +++++++++++++++++++ 12 files changed, 182 insertions(+), 237 deletions(-) delete mode 100644 plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp delete mode 100644 plugins/chain_plugin/test/test_sync_gate.cpp create mode 100644 unittests/controller_sync_tests.cpp diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 99ce81eb8f..7e3f132c51 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4186,6 +4186,14 @@ size_t controller::fork_db_size() const { return my->fork_db_size(); } +bool controller::is_synced(fc::time_point now, fc::microseconds recency_window) const { + return fork_db_has_root() && fork_db_root().block_time() >= now - recency_window; +} + +bool controller::is_synced() const { + return is_synced(fc::time_point::now(), fc::milliseconds(default_sync_recency_ms)); +} + const dynamic_global_property_object& controller::get_dynamic_global_properties()const { return my->db.get(); } diff --git a/libraries/chain/include/sysio/chain/controller.hpp b/libraries/chain/include/sysio/chain/controller.hpp index 96971d9ab0..947cb24565 100644 --- a/libraries/chain/include/sysio/chain/controller.hpp +++ b/libraries/chain/include/sysio/chain/controller.hpp @@ -358,6 +358,45 @@ namespace sysio::chain { // thread-safe size_t fork_db_size() const; + /// Behind-now gap (ms) under which {@link is_synced} counts the node as synced. + /// Must exceed steady-state finality lag (a few blocks) plus scheduling slack, and + /// stay well under one epoch so consumers gating work on the predicate never start + /// against stale state. Not an operator tunable. + static constexpr uint32_t default_sync_recency_ms = 5000; + + /** + * @brief The "node is synced" predicate: the LAST IRREVERSIBLE block's time is + * within `recency_window` of `now`. + * + * LIB recency, not head recency, deliberately: consumers of this predicate + * (operator daemons run read-mode = irreversible) read the IRREVERSIBLE state, and + * head-time recency reports synced while the local LIB still trails the rows those + * reads need (observed live: a registration in block N was readable only 50ms after + * a read at LIB N-3). While a cold-booting node syncs blocks, LIB trails `now` by + * the catch-up gap; once caught up it tracks `now` to within finality-lag jitter. A + * LIB ahead of `now` (clock skew) counts as synced. False while no irreversible + * root exists (fresh boot, mid-replay, snapshot catch-up). + * + * Evaluated on demand, never cached — a cached flag updated on block events would + * go stale-TRUE the moment the chain stalls. There is deliberately no push + * mechanism: the predicate is a LEVEL, and the wake-up a gated consumer needs + * already exists as the `plugin_interface::channels::irreversible_block` channel — + * a LIB advance is the only event that can turn the predicate true, and channel + * deliveries are posted to the application executor (main thread, AFTER the + * triggering block fully commits). Consumers subscribe, re-check per delivery, and + * unsubscribe once it holds. + * + * @param now The instant to measure against (wall-clock in production; + * explicit here so boundary behavior is unit-testable). + * @param recency_window Maximum behind-`now` gap still considered synced. + * @return True when an irreversible root exists and its block time is recent. + */ + // thread-safe + bool is_synced(fc::time_point now, fc::microseconds recency_window) const; + /// {@link is_synced} against wall-clock now at {@link default_sync_recency_ms}. + // thread-safe + bool is_synced() const; + // thread-safe, retrieves block according to fork db best branch which can change at any moment signed_block_ptr fetch_block_by_number( uint32_t block_num )const; // thread-safe diff --git a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp index c4173e7246..cccaa53f9b 100644 --- a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp +++ b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp @@ -190,10 +190,9 @@ struct batch_operator_plugin::impl { std::atomic shutting_down{false}; /// Sync gate: `channels::irreversible_block` subscription that arms - /// {@link run_deferred_startup_or_quit} once `chain_plugin::is_synced()` + /// {@link run_deferred_startup_or_quit} once `controller::is_synced()` /// holds — a LIB advance is the only event that can turn the predicate - /// true (see `sysio/chain_plugin/sync_gate.hpp`). Unsubscribed after - /// arming; released on shutdown. + /// true. Unsubscribed after arming; released on shutdown. chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; // ----------------------------------------------------------------------- @@ -826,7 +825,7 @@ struct batch_operator_plugin::impl { // Sync-gated startup // ----------------------------------------------------------------------- - /// The startup body deferred behind the chain_plugin sync gate: outpost + /// The startup body deferred behind the sync gate: outpost /// discovery → private cron_service creation (sized from the discovered /// outposts) → epoch_tick scheduling → per-outpost relay jobs. Runs on the /// main thread from {@link run_deferred_startup_or_quit} once the node is @@ -964,6 +963,15 @@ void batch_operator_plugin::plugin_initialize(const variables_map& options) { _impl->cron_plug = &app().get_plugin(); _impl->eth_plug = &app().get_plugin(); _impl->sol_plug = &app().get_plugin(); + + // Operator daemons are designed for read-mode = irreversible: the sync gate + // (controller::is_synced) measures LIB recency and every local table read the + // relay performs serves the irreversible view. Any other read mode would relay + // envelopes derived from state that can still fork out. + FC_ASSERT(!_impl->enabled || + _impl->chain_plug->chain().get_read_mode() == chain::db_read_mode::IRREVERSIBLE, + "batch_operator_plugin requires read-mode = irreversible (configured read-mode = {})", + magic_enum::enum_name(_impl->chain_plug->chain().get_read_mode())); } void batch_operator_plugin::plugin_startup() { @@ -978,8 +986,8 @@ void batch_operator_plugin::plugin_startup() { // cold-booting operator node those reads see mid-sync (possibly genesis) // state and throw spuriously, so the whole body (discovery → cron_service → // epoch_tick → relay jobs) is DEFERRED until the node is synced — - // `chain_plugin::is_synced()`: the LAST IRREVERSIBLE block's time within - // `chain_apis::default_sync_recency_ms` of now (the state the reads + // `controller::is_synced()`: the LAST IRREVERSIBLE block's time within + // `controller::default_sync_recency_ms` of now (the state the reads // actually serve under read-mode = irreversible). The wake-up is the // existing `irreversible_block` channel: a LIB advance is the only event // that can turn the predicate true, and channel deliveries are posted to @@ -990,11 +998,6 @@ void batch_operator_plugin::plugin_startup() { // with a producer), and a node that somehow is synced at startup is // released by the next LIB advance. auto& chain = _impl->chain_plug->chain(); - if (chain.get_read_mode() != chain::db_read_mode::IRREVERSIBLE) { - wlog("batch_operator_plugin: node runs read-mode = {}, but operator daemons are designed " - "for read-mode = irreversible — the sync gate and every local table read serve " - "the irreversible state", magic_enum::enum_name(chain.get_read_mode())); - } ilog("batch_operator_plugin: waiting for chain sync before outpost discovery " "(head {} is {}s behind now; irreversible state is {}s behind)", chain.head().block_num(), @@ -1005,7 +1008,7 @@ void batch_operator_plugin::plugin_startup() { _impl->sync_gate_subscription = app().get_channel().subscribe( [impl = _impl.get()](const chain::block_signal_params&) { - if (impl->shutting_down || !impl->chain_plug->is_synced()) { + if (impl->shutting_down || !impl->chain_plug->chain().is_synced()) { return; } // One-shot consumption: unsubscribe (safe from within the slot) and diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp index 2b433900f8..1640c7e149 100644 --- a/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/sysio/chain_plugin/chain_plugin.hpp @@ -695,25 +695,6 @@ class chain_plugin : public plugin { // Only call this after plugin_initialize()! const controller& chain() const; - /// The "node is synced" predicate: true when the LAST IRREVERSIBLE block's time is - /// within `chain_apis::default_sync_recency_ms` of wall-clock now (see - /// `sysio/chain_plugin/sync_gate.hpp`). LIB recency, not head recency, deliberately — - /// the operator-daemon plugins this gates run read-mode = irreversible, so the - /// irreversible state is what their table reads actually serve. False while no - /// irreversible root exists (fresh boot, mid-replay, snapshot catch-up). Computed on - /// demand, never cached, so a stalled chain reads as NOT synced as soon as its LIB - /// leaves the window. - /// - /// Consumers deferring startup until the node is synced subscribe to the existing - /// `chain::plugin_interface::channels::irreversible_block` channel (a LIB advance is - /// the only event that can turn this predicate true), re-check per delivery, and - /// unsubscribe once it holds. Channel deliveries are posted to the application - /// executor, so they run on the main thread AFTER the triggering block fully - /// commits — never mid block-application — and may read chain state directly. - /// - /// Only call this after plugin_initialize()! - bool is_synced() const; - chain::chain_id_type get_chain_id() const; fc::microseconds get_abi_serializer_max_time() const; bool api_accept_transactions() const; diff --git a/plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp b/plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp deleted file mode 100644 index ee6bd73d89..0000000000 --- a/plugins/chain_plugin/include/sysio/chain_plugin/sync_gate.hpp +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once -/** - * @file sync_gate.hpp - * @brief Pure building blocks of the chain_plugin sync gate — the predicate behind - * `chain_plugin::is_synced()` — lifted out of the `.cpp`-private impl so they are - * unit-testable without standing up a chain. - * - * Plugins whose startup validates on-chain state via LOCAL table reads (the operator - * daemons: batch_operator_plugin, underwriter_plugin) must not start until the node is - * synced. On a cold-booting operator node those reads see mid-sync (possibly genesis) - * state and fail spuriously. "Synced" here means the LAST IRREVERSIBLE block's time is - * within a small window of wall-clock now — the irreversible state is what those - * plugins' table reads actually serve (operator daemons run read-mode = irreversible). - * - * There is deliberately no push mechanism here: the predicate is a LEVEL, evaluated on - * demand (never cached — a cached flag updated on block events would go stale-TRUE the - * moment the chain stalls), and the wake-up a gated consumer needs already exists as - * the `chain::plugin_interface::channels::irreversible_block` channel — a LIB advance - * is the only event that can turn the predicate true, and channel deliveries are posted - * to the application executor (main thread, AFTER the triggering block fully commits — - * never mid block-application, where table reads observe an incomplete view). Consumers - * subscribe, re-check `is_synced()` per delivery, and unsubscribe once it holds. - */ - -#include - -#include - -namespace sysio::chain_apis { - - /// Behind-now gap (ms) under which the node counts as synced. Measured against the - /// LAST IRREVERSIBLE block's time — the state operator-daemon table reads actually - /// serve (read-mode = irreversible); see {@link lib_time_is_recent}. Must exceed - /// steady-state finality lag (a few blocks) plus scheduling slack, and stay well - /// under one epoch so gated consumers never start against stale state. Not an - /// operator tunable. - inline constexpr uint32_t default_sync_recency_ms = 5000; - - /** - * @brief True when a block time is recent enough to treat the node as synced. - * - * While a cold-booting node syncs blocks the tested time trails `now` by the - * catch-up gap; once caught up, it tracks `now` to within finality-lag jitter. The - * window must exceed that jitter (plus scheduling slack) but stay well under one - * epoch so gated consumers never start against stale state. - * - * @param block_time The block timestamp to test — the caller's sync criterion - * (the sync gate feeds the LAST IRREVERSIBLE block's time, - * since that is the state operator-daemon reads serve). - * @param now Wall-clock now. - * @param recency_window Maximum behind-now gap still considered synced. - * @return True when `block_time >= now - recency_window`. - */ - inline bool block_time_is_recent(fc::time_point block_time, fc::time_point now, - fc::microseconds recency_window) { - return block_time >= now - recency_window; - } - - /** - * @brief The "node is synced" predicate: the LAST IRREVERSIBLE block's time within - * `recency_window` of `now`. - * - * LIB recency, not head recency, deliberately: operator daemons run read-mode = - * irreversible, so their table reads serve the IRREVERSIBLE state. Head-time - * recency armed the underwriter's original gate while the local LIB still trailed - * the rows its preflight needed (observed live: a registration in block N was - * readable only 50ms after a preflight read at LIB N−3). False while no - * irreversible root exists. - * - * @param chain The controller to read the irreversible root from. - * @param now Wall-clock now. - * @param recency_window Maximum behind-now gap still considered synced. - * @return True when an irreversible root exists and its block time is recent. - */ - inline bool lib_time_is_recent(const chain::controller& chain, fc::time_point now, - fc::microseconds recency_window) { - return chain.fork_db_has_root() && - block_time_is_recent(chain.fork_db_root().block_time(), now, recency_window); - } - -} // namespace sysio::chain_apis diff --git a/plugins/chain_plugin/src/chain_plugin.cpp b/plugins/chain_plugin/src/chain_plugin.cpp index b3a338f7dd..080b2cfcbb 100644 --- a/plugins/chain_plugin/src/chain_plugin.cpp +++ b/plugins/chain_plugin/src/chain_plugin.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -1500,11 +1499,6 @@ void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, controller& chain_plugin::chain() { return *my->chain; } const controller& chain_plugin::chain() const { return *my->chain; } -bool chain_plugin::is_synced() const { - return chain_apis::lib_time_is_recent(chain(), fc::time_point::now(), - fc::milliseconds(chain_apis::default_sync_recency_ms)); -} - chain::chain_id_type chain_plugin::get_chain_id()const { return my->chain->get_chain_id(); } diff --git a/plugins/chain_plugin/test/CMakeLists.txt b/plugins/chain_plugin/test/CMakeLists.txt index dfc2bdc197..9e30f3f891 100644 --- a/plugins/chain_plugin/test/CMakeLists.txt +++ b/plugins/chain_plugin/test/CMakeLists.txt @@ -1,6 +1,5 @@ add_executable( test_chain_plugin test_account_query_db.cpp - test_sync_gate.cpp test_trx_retry_db.cpp test_trx_finality_status_processing.cpp plugin_config_test.cpp @@ -10,7 +9,6 @@ target_link_libraries( test_chain_plugin chain_plugin sysio_testing sysio_chain_ set(CHAIN_PLUGIN_TEST_CASES chain_plugin_default_tests account_query_db_tests - sync_gate_tests trx_finality_status_processing_test trx_retry_db_test ) diff --git a/plugins/chain_plugin/test/test_sync_gate.cpp b/plugins/chain_plugin/test/test_sync_gate.cpp deleted file mode 100644 index 68a77851dc..0000000000 --- a/plugins/chain_plugin/test/test_sync_gate.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/** - * @file test_sync_gate.cpp - * @brief Unit tests for the chain_plugin sync gate (`sysio/chain_plugin/sync_gate.hpp`) - * — the LIB-recency predicate behind `chain_plugin::is_synced()`. - * - * The pure-predicate cases migrated verbatim from the underwriter plugin's private - * gate (`test_underwriter_sync.cpp`) when the gate was promoted to chain_plugin. - */ -#include - -#include -#include - -using namespace sysio::testing; -using sysio::chain_apis::block_time_is_recent; -using sysio::chain_apis::lib_time_is_recent; - -BOOST_AUTO_TEST_SUITE(sync_gate_tests) - -/// The reference "now" every pure case offsets from — an arbitrary fixed instant so -/// the tests are deterministic (no wall clock). -static const fc::time_point reference_now = - fc::time_point{} + fc::days(365 * 50); - -/// The default gate window the plugin runs with. -static const fc::microseconds default_window = - fc::milliseconds(sysio::chain_apis::default_sync_recency_ms); - -// ── block_time_is_recent: the pure recency predicate ── - -BOOST_AUTO_TEST_CASE(head_at_now_is_synced) { - BOOST_TEST(block_time_is_recent(reference_now, reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_slightly_behind_within_window_is_synced) { - // One block interval behind — the steady-state of a caught-up node. - BOOST_TEST(block_time_is_recent(reference_now - fc::milliseconds(500), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_exactly_at_window_boundary_is_synced) { - BOOST_TEST(block_time_is_recent(reference_now - default_window, - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_just_past_window_is_not_synced) { - BOOST_TEST(!block_time_is_recent(reference_now - default_window - fc::microseconds(1), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(cold_boot_replay_gap_is_not_synced) { - // A node mid-replay: head hours behind wall clock — the exact state whose - // genesis-era table reads used to false-fail operator-daemon startups. - BOOST_TEST(!block_time_is_recent(reference_now - fc::hours(2), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_ahead_of_now_is_synced) { - // Clock skew / head stamped marginally ahead of the observer's clock must - // never read as "behind". - BOOST_TEST(block_time_is_recent(reference_now + fc::seconds(1), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(zero_window_requires_head_at_or_past_now) { - BOOST_TEST(block_time_is_recent(reference_now, reference_now, fc::microseconds(0))); - BOOST_TEST(!block_time_is_recent(reference_now - fc::microseconds(1), - reference_now, fc::microseconds(0))); -} - -// ── lib_time_is_recent: the controller-backed predicate `is_synced()` delegates to ── - -BOOST_AUTO_TEST_CASE(stale_chain_is_not_synced) { try { - // The tester genesis timestamp is a fixed instant years in the past, so a - // freshly-produced chain's irreversible root time trails wall-clock now by - // that whole gap — the cold-boot replay state the gate exists to detect. - tester chain; - chain.produce_blocks(4); - BOOST_REQUIRE(chain.control->fork_db_has_root()); - BOOST_TEST(!lib_time_is_recent(*chain.control, fc::time_point::now(), default_window)); -} FC_LOG_AND_RETHROW() } - -BOOST_AUTO_TEST_CASE(fresh_chain_is_synced) { try { - // Jump block time from the fixed genesis instant to wall-clock now, then - // produce a few more blocks so the near-now block becomes irreversible. The - // subsequent block times land at/ahead of now (ahead counts as recent — - // clock-skew rule above), so the LIB-recency predicate must hold. - tester chain; - chain.produce_block(); - chain.produce_block(fc::time_point::now() - chain.control->head().block_time()); - chain.produce_blocks(6); - BOOST_REQUIRE(chain.control->fork_db_has_root()); - BOOST_TEST(lib_time_is_recent(*chain.control, fc::time_point::now(), default_window)); -} FC_LOG_AND_RETHROW() } - -BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp index 0a01bfddb5..3331e20b8e 100644 --- a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp +++ b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp @@ -10,11 +10,10 @@ * registration, chain registry, authex links) via LOCAL table reads. On a * cold-booting operator node those reads see mid-sync (possibly genesis) * state and fail spuriously, so the plugin must not begin underwriting until - * the node is synced. The sync predicate itself is the first-class chain_plugin - * API — `chain_plugin::is_synced()`, backed by - * `sysio/chain_plugin/sync_gate.hpp` and woken by the `irreversible_block` - * channel — shared by every operator-daemon plugin; this header carries only - * the underwriter's OWN gate lifecycle surface. + * the node is synced. The sync predicate itself is `controller::is_synced()` + * (LIB recency, woken by the `irreversible_block` channel), shared by every + * operator-daemon plugin; this header carries only the underwriter's OWN gate + * lifecycle surface. */ #include diff --git a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp index b9303beac1..91bcf05aac 100644 --- a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp +++ b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp @@ -23,9 +23,8 @@ namespace sysio { constexpr uint32_t scan_interval_ms = 5000; constexpr uint32_t action_timeout_ms = 15000; constexpr bool enabled = false; - // The sync-recency window moved to `chain_apis::default_sync_recency_ms` - // (sysio/chain_plugin/sync_gate.hpp) — the sync predicate is the - // first-class chain_plugin API (`is_synced()`), shared by every + // The sync-recency window moved to `controller::default_sync_recency_ms` + // — the sync predicate is `controller::is_synced()`, shared by every // operator-daemon plugin. /// How long after the sync gate arms a failing preflight keeps /// RETRYING before the failure is terminal. Rows the harness confirmed diff --git a/plugins/underwriter_plugin/src/underwriter_plugin.cpp b/plugins/underwriter_plugin/src/underwriter_plugin.cpp index b38d38126c..987bfc729b 100644 --- a/plugins/underwriter_plugin/src/underwriter_plugin.cpp +++ b/plugins/underwriter_plugin/src/underwriter_plugin.cpp @@ -247,10 +247,9 @@ struct underwriter_plugin::impl { std::atomic shutting_down{false}; /// Sync gate: `channels::irreversible_block` subscription that arms - /// {@link run_deferred_startup} once `chain_plugin::is_synced()` holds — a LIB - /// advance is the only event that can turn the predicate true (see - /// `sysio/chain_plugin/sync_gate.hpp`). Unsubscribed after arming; released on - /// shutdown. + /// {@link run_deferred_startup} once `controller::is_synced()` holds — a LIB + /// advance is the only event that can turn the predicate true. Unsubscribed + /// after arming; released on shutdown. chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; /// When the sync gate armed — bounds the preflight retry grace window. fc::time_point startup_armed_at; @@ -753,7 +752,7 @@ struct underwriter_plugin::impl { /// Arm the deferred startup: runs AT MOST once (re-entry is guarded by /// {@link startup_attempted}), on the main thread from the sync-gate channel - /// callback once `chain_plugin::is_synced()` holds. The body itself lives + /// callback once `controller::is_synced()` holds. The body itself lives /// in {@link attempt_deferred_startup}, which may retry the preflight /// within a bounded grace and shuts the node down on terminal failure /// (fail-fast). @@ -2733,6 +2732,15 @@ void underwriter_plugin::plugin_initialize(const variables_map& options) { _impl->cron_plug = &app().get_plugin(); _impl->eth_plug = &app().get_plugin(); _impl->sol_plug = &app().get_plugin(); + + // Operator daemons are designed for read-mode = irreversible: the sync gate + // (controller::is_synced) measures LIB recency and every local table read the + // preflight and scan cycle perform serves the irreversible view. Any other read + // mode would validate/underwrite against state that can still fork out. + FC_ASSERT(!_impl->enabled || + _impl->chain_plug->chain().get_read_mode() == chain::db_read_mode::IRREVERSIBLE, + "underwriter_plugin requires read-mode = irreversible (configured read-mode = {})", + magic_enum::enum_name(_impl->chain_plug->chain().get_read_mode())); } void underwriter_plugin::plugin_startup() { @@ -2757,8 +2765,8 @@ void underwriter_plugin::plugin_startup() { // operator node those reads see mid-sync (possibly genesis) state and // would fail spuriously, so the whole startup body (preflight → outpost // client wiring → cron) is DEFERRED until the node is synced — - // `chain_plugin::is_synced()`: the LAST IRREVERSIBLE block's time within - // `chain_apis::default_sync_recency_ms` of now (the state the reads + // `controller::is_synced()`: the LAST IRREVERSIBLE block's time within + // `controller::default_sync_recency_ms` of now (the state the reads // actually serve under read-mode = irreversible). The wake-up is the // existing `irreversible_block` channel: a LIB advance is the only event // that can turn the predicate true, and channel deliveries are posted to @@ -2773,11 +2781,6 @@ void underwriter_plugin::plugin_startup() { // loudly once the grace expires and shuts the node down (fail-fast) — // there remains no dev escape hatch. auto& chain = _impl->chain_plug->chain(); - if (chain.get_read_mode() != chain::db_read_mode::IRREVERSIBLE) { - wlog("underwriter_plugin: node runs read-mode = {}, but operator daemons are designed " - "for read-mode = irreversible — the sync gate and every local table read serve " - "the irreversible state", magic_enum::enum_name(chain.get_read_mode())); - } ilog("underwriter_plugin: waiting for chain sync before preflight " "(head {} is {}s behind now; irreversible state is {}s behind)", chain.head().block_num(), @@ -2789,7 +2792,7 @@ void underwriter_plugin::plugin_startup() { app().get_channel().subscribe( [this](const chain::block_signal_params&) { if (_impl->startup_attempted() || _impl->shutting_down || - !_impl->chain_plug->is_synced()) { + !_impl->chain_plug->chain().is_synced()) { return; } // One-shot consumption: unsubscribe (safe from within the slot) and diff --git a/unittests/controller_sync_tests.cpp b/unittests/controller_sync_tests.cpp new file mode 100644 index 0000000000..e908866ac5 --- /dev/null +++ b/unittests/controller_sync_tests.cpp @@ -0,0 +1,98 @@ +/** + * @file controller_sync_tests.cpp + * @brief Unit tests for `controller::is_synced()` — the LIB-recency predicate operator-daemon + * plugins gate their deferred startup on. + * + * The boundary cases drive the testable overload `is_synced(now, recency_window)` with instants + * chosen relative to the tester chain's ACTUAL last-irreversible block time, so every comparison + * exercises the real controller code path deterministically (no wall clock). The two wall-clock + * cases at the end cover the production convenience overload. + */ +#include + +#include +#include + +using namespace sysio::chain; +using namespace sysio::testing; + +BOOST_AUTO_TEST_SUITE(controller_sync_tests) + +/// The default gate window the operator-daemon plugins run with. +static const fc::microseconds default_window = + fc::milliseconds(controller::default_sync_recency_ms); + +/// A produced tester chain plus its last-irreversible block time — the reference instant +/// every boundary case below offsets `now` from. +struct lib_fixture { + tester chain; + fc::time_point lib_time; + + lib_fixture() { + chain.produce_blocks(4); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + lib_time = chain.control->fork_db_root().block_time(); + } +}; + +BOOST_FIXTURE_TEST_CASE(lib_at_now_is_synced, lib_fixture) { try { + BOOST_TEST(chain.control->is_synced(lib_time, default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_one_block_interval_behind_is_synced, lib_fixture) { try { + // One block interval behind now — the steady-state of a caught-up node. + BOOST_TEST(chain.control->is_synced(lib_time + fc::milliseconds(500), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_exactly_at_window_boundary_is_synced, lib_fixture) { try { + BOOST_TEST(chain.control->is_synced(lib_time + default_window, default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_just_past_window_is_not_synced, lib_fixture) { try { + BOOST_TEST(!chain.control->is_synced(lib_time + default_window + fc::microseconds(1), + default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(cold_boot_replay_gap_is_not_synced, lib_fixture) { try { + // A node mid-replay: LIB hours behind wall clock — the exact state whose genesis-era + // table reads used to false-fail operator-daemon startups. + BOOST_TEST(!chain.control->is_synced(lib_time + fc::hours(2), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_ahead_of_now_is_synced, lib_fixture) { try { + // Clock skew / LIB stamped marginally ahead of the observer's clock must never read + // as "behind". + BOOST_TEST(chain.control->is_synced(lib_time - fc::seconds(1), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(zero_window_requires_lib_at_or_past_now, lib_fixture) { try { + BOOST_TEST(chain.control->is_synced(lib_time, fc::microseconds(0))); + BOOST_TEST(!chain.control->is_synced(lib_time + fc::microseconds(1), fc::microseconds(0))); +} FC_LOG_AND_RETHROW() } + +// ── the wall-clock convenience overload the plugins call ── + +BOOST_AUTO_TEST_CASE(stale_chain_is_not_synced) { try { + // The tester genesis timestamp is a fixed instant years in the past, so a freshly + // produced chain's irreversible root time trails wall-clock now by that whole gap — + // the cold-boot state the predicate exists to detect. + tester chain; + chain.produce_blocks(4); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + BOOST_TEST(!chain.control->is_synced()); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE(fresh_chain_is_synced) { try { + // Jump block time from the fixed genesis instant to wall-clock now, then produce a few + // more blocks so the near-now block becomes irreversible. The subsequent block times + // land at/ahead of now (ahead counts as recent — clock-skew rule above), so the + // LIB-recency predicate must hold. + tester chain; + chain.produce_block(); + chain.produce_block(fc::time_point::now() - chain.control->head().block_time()); + chain.produce_blocks(6); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + BOOST_TEST(chain.control->is_synced()); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() From 5de3cfe1ff2165b7c972b52551c5767b84e90f08 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 16 Jul 2026 08:02:11 -0500 Subject: [PATCH 3/5] chain, plugins: drop redundant sync-gate shutdown unsubscribe; tighten is_synced doc The channel handle's destructor already unsubscribes at plugin destruction, posted channel deliveries are drained without executing between shutdown_plugins() and destroy_plugins(), and the gate slots check shutting_down first - so the explicit unsubscribe() calls in plugin_shutdown were dead weight (net_plugin's transaction_ack handle follows the same convention). The slot-side one-shot unsubscribe stays. --- .../chain/include/sysio/chain/controller.hpp | 28 ++++--------------- .../src/batch_operator_plugin.cpp | 6 ++-- .../src/underwriter_plugin.cpp | 5 ++-- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/libraries/chain/include/sysio/chain/controller.hpp b/libraries/chain/include/sysio/chain/controller.hpp index 947cb24565..8ee39ad437 100644 --- a/libraries/chain/include/sysio/chain/controller.hpp +++ b/libraries/chain/include/sysio/chain/controller.hpp @@ -365,31 +365,13 @@ namespace sysio::chain { static constexpr uint32_t default_sync_recency_ms = 5000; /** - * @brief The "node is synced" predicate: the LAST IRREVERSIBLE block's time is - * within `recency_window` of `now`. + * @brief True when the LAST IRREVERSIBLE block's time is within `recency_window` + * of `now`; false while no irreversible root exists (fresh boot, + * mid-replay, snapshot catch-up). * - * LIB recency, not head recency, deliberately: consumers of this predicate - * (operator daemons run read-mode = irreversible) read the IRREVERSIBLE state, and - * head-time recency reports synced while the local LIB still trails the rows those - * reads need (observed live: a registration in block N was readable only 50ms after - * a read at LIB N-3). While a cold-booting node syncs blocks, LIB trails `now` by - * the catch-up gap; once caught up it tracks `now` to within finality-lag jitter. A - * LIB ahead of `now` (clock skew) counts as synced. False while no irreversible - * root exists (fresh boot, mid-replay, snapshot catch-up). - * - * Evaluated on demand, never cached — a cached flag updated on block events would - * go stale-TRUE the moment the chain stalls. There is deliberately no push - * mechanism: the predicate is a LEVEL, and the wake-up a gated consumer needs - * already exists as the `plugin_interface::channels::irreversible_block` channel — - * a LIB advance is the only event that can turn the predicate true, and channel - * deliveries are posted to the application executor (main thread, AFTER the - * triggering block fully commits). Consumers subscribe, re-check per delivery, and - * unsubscribe once it holds. - * - * @param now The instant to measure against (wall-clock in production; - * explicit here so boundary behavior is unit-testable). + * @param now Instant to measure against (wall-clock in production; + * explicit for testability). * @param recency_window Maximum behind-`now` gap still considered synced. - * @return True when an irreversible root exists and its block time is recent. */ // thread-safe bool is_synced(fc::time_point now, fc::microseconds recency_window) const; diff --git a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp index cccaa53f9b..620ce90d6e 100644 --- a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp +++ b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp @@ -192,7 +192,10 @@ struct batch_operator_plugin::impl { /// Sync gate: `channels::irreversible_block` subscription that arms /// {@link run_deferred_startup_or_quit} once `controller::is_synced()` /// holds — a LIB advance is the only event that can turn the predicate - /// true. Unsubscribed after arming; released on shutdown. + /// true. Unsubscribed after arming; otherwise the handle's destructor + /// releases it (no shutdown unsubscribe needed: posted channel deliveries + /// are drained without executing after quit, and the slot checks + /// `shutting_down` first). chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; // ----------------------------------------------------------------------- @@ -1024,7 +1027,6 @@ void batch_operator_plugin::plugin_startup() { void batch_operator_plugin::plugin_shutdown() { _impl->shutting_down = true; - _impl->sync_gate_subscription.unsubscribe(); if (_impl->cron_svc) { _impl->cron_svc->cancel_all(); _impl->cron_svc->stop(); diff --git a/plugins/underwriter_plugin/src/underwriter_plugin.cpp b/plugins/underwriter_plugin/src/underwriter_plugin.cpp index 987bfc729b..635c7d72d1 100644 --- a/plugins/underwriter_plugin/src/underwriter_plugin.cpp +++ b/plugins/underwriter_plugin/src/underwriter_plugin.cpp @@ -249,7 +249,9 @@ struct underwriter_plugin::impl { /// Sync gate: `channels::irreversible_block` subscription that arms /// {@link run_deferred_startup} once `controller::is_synced()` holds — a LIB /// advance is the only event that can turn the predicate true. Unsubscribed - /// after arming; released on shutdown. + /// after arming; otherwise the handle's destructor releases it (no shutdown + /// unsubscribe needed: posted channel deliveries are drained without + /// executing after quit, and the slot checks `shutting_down` first). chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; /// When the sync gate armed — bounds the preflight retry grace window. fc::time_point startup_armed_at; @@ -2811,7 +2813,6 @@ void underwriter_plugin::plugin_startup() { void underwriter_plugin::plugin_shutdown() { _impl->shutting_down = true; - _impl->sync_gate_subscription.unsubscribe(); if (_impl->preflight_retry_timer) { _impl->preflight_retry_timer->cancel(); } From f2969824ddd3139618b69b174ce3fe43d13ea737 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 16 Jul 2026 08:22:19 -0500 Subject: [PATCH 4/5] plugins: address sync-gate review nits; cite enforced producer exclusivity Unify the gate-slot capture on the impl pointer, simplify the read-mode assert message to producer_plugin's precedent (dropping batch_operator's now-unused magic_enum include), replace the -1 sentinel in the waiting log with n/a, and document the bad_alloc passthrough on batch_operator's contained startup path. The "never co-hosted with a producer" comments are now statements of configuration fact rather than topology convention: these daemons require read-mode = irreversible and producer_plugin rejects producer-names under irreversible read-mode, so the comments cite that enforcement. --- .../src/batch_operator_plugin.cpp | 22 ++++++++------- .../src/underwriter_plugin.cpp | 27 ++++++++++--------- unittests/controller_sync_tests.cpp | 7 +++-- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp index 620ce90d6e..da478d2b8e 100644 --- a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp +++ b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -897,7 +896,9 @@ struct batch_operator_plugin::impl { /// (outpost discovery against a not-yet-deployed registry) are already /// absorbed inside {@link run_deferred_startup} and retried by the refresh /// ticks; what reaches here is structural (cron_service creation or job - /// scheduling failed). + /// scheduling failed). FC_LOG_AND_DROP deliberately rethrows + /// boost::interprocess::bad_alloc — chainbase shared-memory exhaustion + /// stays immediately fatal. void run_deferred_startup_or_quit() { try { run_deferred_startup(); @@ -970,11 +971,13 @@ void batch_operator_plugin::plugin_initialize(const variables_map& options) { // Operator daemons are designed for read-mode = irreversible: the sync gate // (controller::is_synced) measures LIB recency and every local table read the // relay performs serves the irreversible view. Any other read mode would relay - // envelopes derived from state that can still fork out. + // envelopes derived from state that can still fork out. Together with + // producer_plugin's inverse assert (no producer-name under irreversible + // read-mode), this also makes co-hosting a producer with an operator daemon + // impossible by configuration. FC_ASSERT(!_impl->enabled || _impl->chain_plug->chain().get_read_mode() == chain::db_read_mode::IRREVERSIBLE, - "batch_operator_plugin requires read-mode = irreversible (configured read-mode = {})", - magic_enum::enum_name(_impl->chain_plug->chain().get_read_mode())); + "batch_operator_plugin requires read-mode = irreversible"); } void batch_operator_plugin::plugin_startup() { @@ -997,8 +1000,9 @@ void batch_operator_plugin::plugin_startup() { // the application executor — main thread, AFTER the triggering block fully // commits — so the callback may run the startup body directly. There is // deliberately no already-synced fast path: operator daemons boot with - // genesis-stale LIB in every deployment topology (they are never co-hosted - // with a producer), and a node that somehow is synced at startup is + // genesis-stale LIB in every deployment topology (producer co-hosting is + // impossible by configuration — see the read-mode requirement in + // plugin_initialize), and a node that somehow is synced at startup is // released by the next LIB advance. auto& chain = _impl->chain_plug->chain(); ilog("batch_operator_plugin: waiting for chain sync before outpost discovery " @@ -1006,8 +1010,8 @@ void batch_operator_plugin::plugin_startup() { chain.head().block_num(), (fc::time_point::now() - chain.head().block_time()).to_seconds(), chain.fork_db_has_root() - ? (fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds() - : -1); + ? std::to_string((fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds()) + : "n/a"); _impl->sync_gate_subscription = app().get_channel().subscribe( [impl = _impl.get()](const chain::block_signal_params&) { diff --git a/plugins/underwriter_plugin/src/underwriter_plugin.cpp b/plugins/underwriter_plugin/src/underwriter_plugin.cpp index 635c7d72d1..13cd8b60d7 100644 --- a/plugins/underwriter_plugin/src/underwriter_plugin.cpp +++ b/plugins/underwriter_plugin/src/underwriter_plugin.cpp @@ -2738,11 +2738,13 @@ void underwriter_plugin::plugin_initialize(const variables_map& options) { // Operator daemons are designed for read-mode = irreversible: the sync gate // (controller::is_synced) measures LIB recency and every local table read the // preflight and scan cycle perform serves the irreversible view. Any other read - // mode would validate/underwrite against state that can still fork out. + // mode would validate/underwrite against state that can still fork out. Together + // with producer_plugin's inverse assert (no producer-name under irreversible + // read-mode), this also makes co-hosting a producer with an operator daemon + // impossible by configuration. FC_ASSERT(!_impl->enabled || _impl->chain_plug->chain().get_read_mode() == chain::db_read_mode::IRREVERSIBLE, - "underwriter_plugin requires read-mode = irreversible (configured read-mode = {})", - magic_enum::enum_name(_impl->chain_plug->chain().get_read_mode())); + "underwriter_plugin requires read-mode = irreversible"); } void underwriter_plugin::plugin_startup() { @@ -2775,8 +2777,9 @@ void underwriter_plugin::plugin_startup() { // the application executor — main thread, AFTER the triggering block // fully commits — so the callback may run the startup body directly. // There is deliberately no already-synced fast path: operator daemons - // boot with genesis-stale LIB in every deployment topology (they are - // never co-hosted with a producer), and a node that somehow is synced at + // boot with genesis-stale LIB in every deployment topology (producer + // co-hosting is impossible by configuration — see the read-mode + // requirement in plugin_initialize), and a node that somehow is synced at // startup is released by the next LIB advance. Post-arming, the preflight // retries within a bounded grace (see attempt_deferred_startup) to absorb // the LIB boundary race; a genuinely incomplete bootstrap still fails @@ -2788,13 +2791,13 @@ void underwriter_plugin::plugin_startup() { chain.head().block_num(), (fc::time_point::now() - chain.head().block_time()).to_seconds(), chain.fork_db_has_root() - ? (fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds() - : -1); + ? std::to_string((fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds()) + : "n/a"); _impl->sync_gate_subscription = app().get_channel().subscribe( - [this](const chain::block_signal_params&) { - if (_impl->startup_attempted() || _impl->shutting_down || - !_impl->chain_plug->chain().is_synced()) { + [impl = _impl.get()](const chain::block_signal_params&) { + if (impl->startup_attempted() || impl->shutting_down || + !impl->chain_plug->chain().is_synced()) { return; } // One-shot consumption: unsubscribe (safe from within the slot) and @@ -2805,9 +2808,9 @@ void underwriter_plugin::plugin_startup() { // block-application, table reads see an incomplete view — observed // live: a registered operator row read back EMPTY → spurious // preflight failure). - _impl->sync_gate_subscription.unsubscribe(); + impl->sync_gate_subscription.unsubscribe(); ilog("underwriter_plugin: chain synced — starting deferred startup"); - _impl->run_deferred_startup(); + impl->run_deferred_startup(); }); } diff --git a/unittests/controller_sync_tests.cpp b/unittests/controller_sync_tests.cpp index e908866ac5..ce036e57a6 100644 --- a/unittests/controller_sync_tests.cpp +++ b/unittests/controller_sync_tests.cpp @@ -18,15 +18,13 @@ using namespace sysio::testing; BOOST_AUTO_TEST_SUITE(controller_sync_tests) -/// The default gate window the operator-daemon plugins run with. -static const fc::microseconds default_window = - fc::milliseconds(controller::default_sync_recency_ms); - /// A produced tester chain plus its last-irreversible block time — the reference instant /// every boundary case below offsets `now` from. struct lib_fixture { tester chain; fc::time_point lib_time; + /// The default gate window the operator-daemon plugins run with. + fc::microseconds default_window = fc::milliseconds(controller::default_sync_recency_ms); lib_fixture() { chain.produce_blocks(4); @@ -36,6 +34,7 @@ struct lib_fixture { }; BOOST_FIXTURE_TEST_CASE(lib_at_now_is_synced, lib_fixture) { try { + // The identity point of the window arithmetic: now == LIB time (zero gap). BOOST_TEST(chain.control->is_synced(lib_time, default_window)); } FC_LOG_AND_RETHROW() } From f17feb82727c1a8b5927861544b868fa8d191733 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Thu, 16 Jul 2026 09:14:59 -0500 Subject: [PATCH 5/5] plugins: document why the sync gate has no inline is_synced check at startup A LIB advance is both the gate's wake-up and the daemons' work supply: everything the operator daemons act on is newly-finalized state, so a finality stall that spans startup (the classic lost-wakeup case) is one with nothing to relay or underwrite - the first finalized block re-arms the gate and creates the first actionable state at the same instant. --- .../batch_operator_plugin/src/batch_operator_plugin.cpp | 6 +++++- plugins/underwriter_plugin/src/underwriter_plugin.cpp | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp index da478d2b8e..3eece09117 100644 --- a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp +++ b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp @@ -1003,7 +1003,11 @@ void batch_operator_plugin::plugin_startup() { // genesis-stale LIB in every deployment topology (producer co-hosting is // impossible by configuration — see the read-mode requirement in // plugin_initialize), and a node that somehow is synced at startup is - // released by the next LIB advance. + // released by the next LIB advance. That advance is also the relay's WORK + // SUPPLY — everything here acts on newly-finalized state — so a finality + // stall that spans startup (the classic lost-wakeup case) is one with + // nothing to relay: the first finalized block re-arms the gate and + // creates the first actionable state at the same instant. auto& chain = _impl->chain_plug->chain(); ilog("batch_operator_plugin: waiting for chain sync before outpost discovery " "(head {} is {}s behind now; irreversible state is {}s behind)", diff --git a/plugins/underwriter_plugin/src/underwriter_plugin.cpp b/plugins/underwriter_plugin/src/underwriter_plugin.cpp index 13cd8b60d7..aa77e922ca 100644 --- a/plugins/underwriter_plugin/src/underwriter_plugin.cpp +++ b/plugins/underwriter_plugin/src/underwriter_plugin.cpp @@ -2780,7 +2780,12 @@ void underwriter_plugin::plugin_startup() { // boot with genesis-stale LIB in every deployment topology (producer // co-hosting is impossible by configuration — see the read-mode // requirement in plugin_initialize), and a node that somehow is synced at - // startup is released by the next LIB advance. Post-arming, the preflight + // startup is released by the next LIB advance. That advance is also the + // underwriter's WORK SUPPLY — everything here acts on newly-finalized + // state — so a finality stall that spans startup (the classic lost-wakeup + // case) is one with nothing to underwrite: the first finalized block + // re-arms the gate and creates the first actionable state at the same + // instant. Post-arming, the preflight // retries within a bounded grace (see attempt_deferred_startup) to absorb // the LIB boundary race; a genuinely incomplete bootstrap still fails // loudly once the grace expires and shuts the node down (fail-fast) —