diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 99ce81eb8f..7e3f132c51 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4186,6 +4186,14 @@ size_t controller::fork_db_size() const { return my->fork_db_size(); } +bool controller::is_synced(fc::time_point now, fc::microseconds recency_window) const { + return fork_db_has_root() && fork_db_root().block_time() >= now - recency_window; +} + +bool controller::is_synced() const { + return is_synced(fc::time_point::now(), fc::milliseconds(default_sync_recency_ms)); +} + const dynamic_global_property_object& controller::get_dynamic_global_properties()const { return my->db.get(); } diff --git a/libraries/chain/include/sysio/chain/controller.hpp b/libraries/chain/include/sysio/chain/controller.hpp index 96971d9ab0..8ee39ad437 100644 --- a/libraries/chain/include/sysio/chain/controller.hpp +++ b/libraries/chain/include/sysio/chain/controller.hpp @@ -358,6 +358,27 @@ namespace sysio::chain { // thread-safe size_t fork_db_size() const; + /// Behind-now gap (ms) under which {@link is_synced} counts the node as synced. + /// Must exceed steady-state finality lag (a few blocks) plus scheduling slack, and + /// stay well under one epoch so consumers gating work on the predicate never start + /// against stale state. Not an operator tunable. + static constexpr uint32_t default_sync_recency_ms = 5000; + + /** + * @brief True when the LAST IRREVERSIBLE block's time is within `recency_window` + * of `now`; false while no irreversible root exists (fresh boot, + * mid-replay, snapshot catch-up). + * + * @param now Instant to measure against (wall-clock in production; + * explicit for testability). + * @param recency_window Maximum behind-`now` gap still considered synced. + */ + // thread-safe + bool is_synced(fc::time_point now, fc::microseconds recency_window) const; + /// {@link is_synced} against wall-clock now at {@link default_sync_recency_ms}. + // thread-safe + bool is_synced() const; + // thread-safe, retrieves block according to fork db best branch which can change at any moment signed_block_ptr fetch_block_by_number( uint32_t block_num )const; // thread-safe diff --git a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp index 1575327b2f..3eece09117 100644 --- a/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp +++ b/plugins/batch_operator_plugin/src/batch_operator_plugin.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -187,6 +188,15 @@ struct batch_operator_plugin::impl { std::map scheduled_opp_jobs; std::atomic shutting_down{false}; + /// Sync gate: `channels::irreversible_block` subscription that arms + /// {@link run_deferred_startup_or_quit} once `controller::is_synced()` + /// holds — a LIB advance is the only event that can turn the predicate + /// true. Unsubscribed after arming; otherwise the handle's destructor + /// releases it (no shutdown unsubscribe needed: posted channel deliveries + /// are drained without executing after quit, and the slot checks + /// `shutting_down` first). + chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; + // ----------------------------------------------------------------------- // Chain-agnostic orchestration layer // ----------------------------------------------------------------------- @@ -812,6 +822,91 @@ struct batch_operator_plugin::impl { elog("batch_operator: push {}::{} timed out", contract, action_name); } } + + // ----------------------------------------------------------------------- + // Sync-gated startup + // ----------------------------------------------------------------------- + + /// The startup body deferred behind the sync gate: outpost + /// discovery → private cron_service creation (sized from the discovered + /// outposts) → epoch_tick scheduling → per-outpost relay jobs. Runs on the + /// main thread from {@link run_deferred_startup_or_quit} once the node is + /// synced. Deferral exists because `refresh_outposts` reads `sysio.chains` + /// LOCALLY: on a cold-booting operator node still replaying toward the + /// deploy blocks the read throws Account/Contract Query Exceptions + /// (3060002/3060003) — spurious boot-window errors the gate removes. + void run_deferred_startup() { + if (shutting_down) { + return; + } + + // Discover outposts before the private cron_service starts. Later refresh + // ticks add/remove per-outpost cron jobs as the active chain set changes. + try { + refresh_outposts(); + } catch (const fc::exception& e) { + wlog("batch_operator_plugin: initial outpost discovery failed: {}. " + "Starting with 0 per-outpost jobs; refresh ticks will retry after " + "the chain has caught up.", e.to_string()); + } + + // Size the pool for startup outposts. Later dynamic outposts are added to + // the same queued cron service; the minimum keeps epoch_tick viable even + // when no outposts are known yet. + const std::size_t outpost_count = opp_jobs.size(); + const std::size_t required_threads = outpost_count * OPP_CRON_JOBS_PER_OUTPOST + EPOCH_TICK_CRON_JOBS; + const std::size_t thread_count = std::max(required_threads, MIN_CRON_THREADS); + + sysio::services::cron_service::options svc_opts; + svc_opts.name = "batch_operator"; + svc_opts.num_threads = thread_count; + svc_opts.autostart = true; + + cron_svc = sysio::services::cron_service::create(svc_opts); + ilog("batch_operator_plugin: cron_service started with {} thread(s) ({} outpost(s) discovered)", + thread_count, outpost_count); + + const auto poll_ms = epoch_poll_ms; + + // epoch_tick — refresh epoch state + election. Keeps `current_epoch` + // and `within_epoch_window` accurate for every per-outpost job. + { + sysio::services::cron_service::job_schedule sched; + sched.milliseconds = {sysio::services::cron_service::job_schedule::step_value{poll_ms}}; + sysio::services::cron_service::job_metadata_t meta; + meta.label = "batch_operator_epoch_tick"; + meta.one_at_a_time = true; + auto id = cron_svc->add(sched, + [this]() { poll_epoch_state(); }, + meta); + cron_job_ids.push_back(id); + ilog("batch_operator_plugin: scheduled {} (id={}, every {}ms)", meta.label, id, poll_ms); + } + + schedule_opp_jobs(); + } + + /// {@link run_deferred_startup} plus the uniform fail-fast policy: the + /// sync-gate callback is a posted channel delivery, so an escaping + /// exception would unwind the application executor mid-task with no + /// diagnosable trace of WHAT failed. Contain it long enough to log, then + /// shut the node down — an operator daemon whose relay never started must + /// be supervisor-visible (it has liveness/slashing consequences), not + /// hidden behind a running process. Expected transient failures + /// (outpost discovery against a not-yet-deployed registry) are already + /// absorbed inside {@link run_deferred_startup} and retried by the refresh + /// ticks; what reaches here is structural (cron_service creation or job + /// scheduling failed). FC_LOG_AND_DROP deliberately rethrows + /// boost::interprocess::bad_alloc — chainbase shared-memory exhaustion + /// stays immediately fatal. + void run_deferred_startup_or_quit() { + try { + run_deferred_startup(); + return; + } FC_LOG_AND_DROP("batch_operator_plugin: deferred startup failed unexpectedly:"); + elog("batch_operator_plugin: deferred startup failed terminally — shutting down node (fail-fast)"); + app().quit(); + } }; // --------------------------------------------------------------------------- @@ -872,6 +967,17 @@ void batch_operator_plugin::plugin_initialize(const variables_map& options) { _impl->cron_plug = &app().get_plugin(); _impl->eth_plug = &app().get_plugin(); _impl->sol_plug = &app().get_plugin(); + + // Operator daemons are designed for read-mode = irreversible: the sync gate + // (controller::is_synced) measures LIB recency and every local table read the + // relay performs serves the irreversible view. Any other read mode would relay + // envelopes derived from state that can still fork out. Together with + // producer_plugin's inverse assert (no producer-name under irreversible + // read-mode), this also makes co-hosting a producer with an operator daemon + // impossible by configuration. + FC_ASSERT(!_impl->enabled || + _impl->chain_plug->chain().get_read_mode() == chain::db_read_mode::IRREVERSIBLE, + "batch_operator_plugin requires read-mode = irreversible"); } void batch_operator_plugin::plugin_startup() { @@ -882,50 +988,49 @@ void batch_operator_plugin::plugin_startup() { ilog("batch_operator_plugin: starting for account {}", _impl->operator_account.to_string()); - // Discover outposts before the private cron_service starts. Later refresh - // ticks add/remove per-outpost cron jobs as the active chain set changes. - try { - _impl->refresh_outposts(); - } catch (const fc::exception& e) { - wlog("batch_operator_plugin: initial outpost discovery failed: {}. " - "Starting with 0 per-outpost jobs; refresh ticks will retry after " - "the chain has caught up.", e.to_string()); - } - - // Size the pool for startup outposts. Later dynamic outposts are added to - // the same queued cron service; the minimum keeps epoch_tick viable even - // when no outposts are known yet. - const std::size_t outpost_count = _impl->opp_jobs.size(); - const std::size_t required_threads = outpost_count * OPP_CRON_JOBS_PER_OUTPOST + EPOCH_TICK_CRON_JOBS; - const std::size_t thread_count = std::max(required_threads, MIN_CRON_THREADS); - - sysio::services::cron_service::options svc_opts; - svc_opts.name = "batch_operator"; - svc_opts.num_threads = thread_count; - svc_opts.autostart = true; - - _impl->cron_svc = sysio::services::cron_service::create(svc_opts); - ilog("batch_operator_plugin: cron_service started with {} thread(s) ({} outpost(s) discovered)", - thread_count, outpost_count); - - const auto poll_ms = _impl->epoch_poll_ms; - - // epoch_tick — refresh epoch state + election. Keeps `current_epoch` - // and `within_epoch_window` accurate for every per-outpost job. - { - sysio::services::cron_service::job_schedule sched; - sched.milliseconds = {sysio::services::cron_service::job_schedule::step_value{poll_ms}}; - sysio::services::cron_service::job_metadata_t meta; - meta.label = "batch_operator_epoch_tick"; - meta.one_at_a_time = true; - auto id = _impl->cron_svc->add(sched, - [impl = _impl.get()]() { impl->poll_epoch_state(); }, - meta); - _impl->cron_job_ids.push_back(id); - ilog("batch_operator_plugin: scheduled {} (id={}, every {}ms)", meta.label, id, poll_ms); - } - - _impl->schedule_opp_jobs(); + // The startup body's outpost discovery reads `sysio.chains` LOCALLY. On a + // cold-booting operator node those reads see mid-sync (possibly genesis) + // state and throw spuriously, so the whole body (discovery → cron_service → + // epoch_tick → relay jobs) is DEFERRED until the node is synced — + // `controller::is_synced()`: the LAST IRREVERSIBLE block's time within + // `controller::default_sync_recency_ms` of now (the state the reads + // actually serve under read-mode = irreversible). The wake-up is the + // existing `irreversible_block` channel: a LIB advance is the only event + // that can turn the predicate true, and channel deliveries are posted to + // the application executor — main thread, AFTER the triggering block fully + // commits — so the callback may run the startup body directly. There is + // deliberately no already-synced fast path: operator daemons boot with + // genesis-stale LIB in every deployment topology (producer co-hosting is + // impossible by configuration — see the read-mode requirement in + // plugin_initialize), and a node that somehow is synced at startup is + // released by the next LIB advance. That advance is also the relay's WORK + // SUPPLY — everything here acts on newly-finalized state — so a finality + // stall that spans startup (the classic lost-wakeup case) is one with + // nothing to relay: the first finalized block re-arms the gate and + // creates the first actionable state at the same instant. + auto& chain = _impl->chain_plug->chain(); + ilog("batch_operator_plugin: waiting for chain sync before outpost discovery " + "(head {} is {}s behind now; irreversible state is {}s behind)", + chain.head().block_num(), + (fc::time_point::now() - chain.head().block_time()).to_seconds(), + chain.fork_db_has_root() + ? std::to_string((fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds()) + : "n/a"); + _impl->sync_gate_subscription = + app().get_channel().subscribe( + [impl = _impl.get()](const chain::block_signal_params&) { + if (impl->shutting_down || !impl->chain_plug->chain().is_synced()) { + return; + } + // One-shot consumption: unsubscribe (safe from within the slot) and + // run the startup body directly — channel deliveries are posted to + // the application executor, so this already runs on the main thread + // AFTER the triggering block committed (mid block-application, + // table reads would observe an incomplete view). + impl->sync_gate_subscription.unsubscribe(); + ilog("batch_operator_plugin: chain synced — starting deferred startup"); + impl->run_deferred_startup_or_quit(); + }); } void batch_operator_plugin::plugin_shutdown() { diff --git a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp index e1a837bcf2..3331e20b8e 100644 --- a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp +++ b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/sync_detail.hpp @@ -1,17 +1,19 @@ #pragma once /** * @file sync_detail.hpp - * @brief Pure sync-gate predicate for the underwriter plugin, lifted out of the - * `.cpp`-private impl so it is unit-testable without standing up a chain. + * @brief Pure gate-lifecycle machinery for the underwriter plugin — the deferred- + * startup states and the `/v1/underwriter/*` gate payloads — lifted out of + * the `.cpp`-private impl so they are unit-testable without standing up a + * chain. * * The underwriter's startup preflight validates depot-side state (opreg * registration, chain registry, authex links) via LOCAL table reads. On a * cold-booting operator node those reads see mid-sync (possibly genesis) * state and fail spuriously, so the plugin must not begin underwriting until - * the node is synced. "Synced" here means the LAST IRREVERSIBLE block's time - * is within a small window of wall-clock now — the irreversible state is what - * the plugin's table reads actually serve (operator daemons run read-mode = - * irreversible). + * the node is synced. The sync predicate itself is `controller::is_synced()` + * (LIB recency, woken by the `irreversible_block` channel), shared by every + * operator-daemon plugin; this header carries only the underwriter's OWN gate + * lifecycle surface. */ #include @@ -23,26 +25,6 @@ namespace sysio::underwriter_detail { -/** - * @brief True when a block time is recent enough to treat the node as synced. - * - * While a cold-booting node syncs blocks the tested time trails `now` by the - * catch-up gap; once caught up, it tracks `now` to within finality-lag - * jitter. The window must exceed that jitter (plus scheduling slack) but stay - * well under one epoch so underwriting never starts against stale state. - * - * @param block_time The block timestamp to test — the caller's sync - * criterion (the underwriter feeds the LAST IRREVERSIBLE - * block's time, since that is the state its reads serve). - * @param now Wall-clock now. - * @param recency_window Maximum behind-now gap still considered synced. - * @return True when `block_time >= now - recency_window`. - */ -inline bool block_time_is_recent(fc::time_point block_time, fc::time_point now, - fc::microseconds recency_window) { - return block_time >= now - recency_window; -} - /// JSON field keys shared by the gate payloads below, the post-startup /// response builders in `underwriter_plugin.cpp`, and their tests — one /// spelling authority instead of per-translation-unit string literals. @@ -73,13 +55,30 @@ enum class startup_state : uint8_t { wiring_failed, ///< Preflight passed but outpost client wiring threw (terminal). startup_failed, ///< The deferred startup body threw past the specific ///< failure paths above (terminal). Exists so an escaping - ///< exception is contained as a diagnosable gate state - ///< instead of unwinding the app executor — which would - ///< tear down the whole node — with the gate still - ///< claiming `waiting_for_sync`. + ///< exception is contained as a diagnosable gate state — + ///< stored and logged BEFORE the fail-fast shutdown — instead + ///< of unwinding the app executor mid-task with the gate + ///< still claiming `waiting_for_sync`. active ///< Deferred startup completed — the scan cron is scheduled. }; +/** + * @brief True for the terminal deferred-startup FAILURE states — the states the + * uniform fail-fast policy shuts the node down on (see + * `underwriter_plugin.cpp`'s `quit_if_startup_failed_terminally`). + * + * `active` is terminal success, not failure; `preflight_retrying` is transient by + * definition (bounded by the retry grace). Kept beside the enum so a new state + * cannot be added without deciding its fail-fast classification here. + * + * @param s The deferred-startup lifecycle state to classify. + * @return True when `s` is a terminal failure state. + */ +inline constexpr bool is_terminal_failure(startup_state s) { + return s == startup_state::preflight_failed || s == startup_state::wiring_failed || + s == startup_state::startup_failed; +} + /// The wire `status` spelling of {@link startup_state::active} — shared by the /// post-startup response builders so the value has one authority beside the /// enum whose member spellings define the wire format. @@ -91,15 +90,15 @@ static_assert(active_status == "active"); /// `detail` carried by the {@link startup_state::preflight_failed} gate payload. inline constexpr std::string_view preflight_failed_detail = "startup preflight failed after sync — depot-side state for this underwriter " - "is incomplete; underwriting is disabled (see node log)"; + "is incomplete; the node is shutting down (fail-fast; see node log)"; /// `detail` carried by the {@link startup_state::wiring_failed} gate payload. inline constexpr std::string_view wiring_failed_detail = - "outpost client wiring failed after preflight; underwriting is disabled " - "(see node log)"; + "outpost client wiring failed after preflight; the node is shutting down " + "(fail-fast; see node log)"; /// `detail` carried by the {@link startup_state::startup_failed} gate payload. inline constexpr std::string_view startup_failed_detail = - "deferred startup failed unexpectedly; underwriting is disabled " - "(see node log)"; + "deferred startup failed unexpectedly; the node is shutting down " + "(fail-fast; see node log)"; /** * @brief The JSON body a `/v1/underwriter/...` endpoint answers with for `state`. diff --git a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp index abc0d7f140..91bcf05aac 100644 --- a/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp +++ b/plugins/underwriter_plugin/include/sysio/underwriter_plugin/underwriter_plugin.hpp @@ -23,14 +23,9 @@ namespace sysio { constexpr uint32_t scan_interval_ms = 5000; constexpr uint32_t action_timeout_ms = 15000; constexpr bool enabled = false; - /// Behind-now gap (ms) under which the node counts as synced. Measured - /// against the LAST IRREVERSIBLE block's time — the state the plugin's - /// table reads actually serve (operator daemons run read-mode = - /// irreversible). Gates the deferred startup (preflight → cron) on - /// cold-booting operator nodes; see `sync_detail.hpp::block_time_is_recent`. - /// Must exceed steady-state finality lag (a few blocks) and stay well - /// under one epoch. - constexpr uint32_t sync_recency_ms = 5000; + // The sync-recency window moved to `controller::default_sync_recency_ms` + // — the sync predicate is `controller::is_synced()`, shared by every + // operator-daemon plugin. /// How long after the sync gate arms a failing preflight keeps /// RETRYING before the failure is terminal. Rows the harness confirmed /// final on the producer can land in the LOCAL irreversible state a diff --git a/plugins/underwriter_plugin/src/underwriter_plugin.cpp b/plugins/underwriter_plugin/src/underwriter_plugin.cpp index e1bbc3d07a..aa77e922ca 100644 --- a/plugins/underwriter_plugin/src/underwriter_plugin.cpp +++ b/plugins/underwriter_plugin/src/underwriter_plugin.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -245,11 +246,13 @@ struct underwriter_plugin::impl { cron_service::job_id_t scan_job_id = 0; std::atomic shutting_down{false}; - /// Sync gate: `accepted_block` subscription that arms {@link run_deferred_startup} - /// once the local irreversible state is current (see - /// `sync_detail.hpp::block_time_is_recent`). Disconnected after arming; reset on - /// shutdown. - std::optional sync_gate_connection; + /// Sync gate: `channels::irreversible_block` subscription that arms + /// {@link run_deferred_startup} once `controller::is_synced()` holds — a LIB + /// advance is the only event that can turn the predicate true. Unsubscribed + /// after arming; otherwise the handle's destructor releases it (no shutdown + /// unsubscribe needed: posted channel deliveries are drained without + /// executing after quit, and the slot checks `shutting_down` first). + chain::plugin_interface::channels::irreversible_block::channel_type::handle sync_gate_subscription; /// When the sync gate armed — bounds the preflight retry grace window. fc::time_point startup_armed_at; /// Bounded-grace preflight retry timer (main io_context; main-thread wait). @@ -749,48 +752,19 @@ struct underwriter_plugin::impl { // Sync-gated startup // ----------------------------------------------------------------------- - /// The "node is synced" predicate: the LAST IRREVERSIBLE block's time - /// within `underwriter_defaults::sync_recency_ms` of wall-clock now. - /// The plugin's table reads serve the IRREVERSIBLE state (operator - /// daemons run read-mode = irreversible), so the gate must measure LIB - /// recency — head-time recency armed the gate while the local LIB still - /// trailed the rows the preflight needs (observed live: a registration - /// in block N was readable only 50ms after a preflight read at LIB N−3). - /// While a cold-booting operator node syncs blocks, LIB trails `now` by - /// the catch-up gap and every local table read sees stale (possibly - /// genesis) state — preflight must not run until this is true. - bool chain_is_synced() const { - auto& chain = chain_plug->chain(); - if (!chain.fork_db_has_root()) { - return false; - } - return underwriter_detail::block_time_is_recent( - chain.fork_db_root().block_time(), fc::time_point::now(), - fc::milliseconds(underwriter_defaults::sync_recency_ms)); - } - - /// What an exception escaping the deferred startup body does. The - /// synchronous `plugin_startup` path must ABORT node startup — fail-fast, - /// supervisor-visible; booting without underwriting would hide a broken - /// deployment behind one log line. Tasks posted after `exec()` is live - /// must CONTAIN the escape as a terminal gate state instead: unwinding - /// out of a posted task makes appbase `quit()` and tear down a node that - /// may be serving other roles. - enum class escape_policy : uint8_t { abort_on_escape, contain_escapes }; - /// Arm the deferred startup: runs AT MOST once (re-entry is guarded by - /// {@link startup_attempted}), on the main thread (either directly from - /// `plugin_startup` when the node is already synced, or from the - /// `accepted_block` signal once it becomes synced). The body itself lives + /// {@link startup_attempted}), on the main thread from the sync-gate channel + /// callback once `controller::is_synced()` holds. The body itself lives /// in {@link attempt_deferred_startup}, which may retry the preflight - /// within a bounded grace. - void run_deferred_startup(escape_policy policy) { + /// within a bounded grace and shuts the node down on terminal failure + /// (fail-fast). + void run_deferred_startup() { startup_armed_at = fc::time_point::now(); - // Release the gate slot; the callback already disconnected it before - // posting, so this is an idempotent cleanup, not the load-bearing - // disconnect. - sync_gate_connection.reset(); - attempt_deferred_startup(policy); + // Release the gate subscription; the callback already unsubscribed + // before calling here, so this is an idempotent cleanup, not the + // load-bearing disconnect. + sync_gate_subscription.unsubscribe(); + attempt_deferred_startup(); } /// One attempt of the deferred startup body: preflight → outpost client @@ -801,44 +775,57 @@ struct underwriter_plugin::impl { /// (observed live: `regoperator` in block 405 readable 50ms after a /// preflight read at LIB 402), so the attempt re-schedules itself on /// `preflight_retry_interval_ms` until the grace expires. Past the grace, - /// a preflight failure on a synced chain remains a loud, terminal - /// bootstrap bug — no cron job, no dev escape hatch — exactly as before. - void attempt_deferred_startup(escape_policy policy) { + /// a preflight failure on a synced chain is a terminal bootstrap bug — + /// stored as a diagnosable gate state, logged, and answered with a + /// fail-fast node shutdown (see {@link quit_if_startup_failed_terminally}). + /// + /// Escapes are contained FIRST (unwinding out of a posted channel delivery + /// would skip the diagnosable gate-state store and the shutdown log) and + /// then routed through the same fail-fast shutdown. The expected + /// preflight/wiring failures inside the body set their own more specific + /// states; the containment catches what they don't (a throwing table + /// decode in the preflight or registry read, a non-fc exception from + /// client wiring, a cron add_job failure). FC_LOG_AND_DROP is the tree's + /// containment idiom (see scan_cycle below) and deliberately rethrows + /// boost::interprocess::bad_alloc — chainbase shared-memory exhaustion + /// stays immediately fatal. + void attempt_deferred_startup() { if (shutting_down) { return; } - if (policy == escape_policy::abort_on_escape) { - // Synchronous plugin_startup path: let an escape abort node startup - // (nodeop exits nonzero) exactly as before the sync gate existed. + bool completed = false; + try { attempt_deferred_startup_body(); - } else { - // Posted contexts (the sync-gate callback's task and the preflight - // retry timer): contain every escape as a terminal, diagnosable gate - // state — unwinding out of a posted task makes appbase quit() and - // tear down the whole node, with the gate still claiming - // waiting_for_sync. The expected preflight/wiring failures inside - // the body set their own more specific states; this catches what - // they don't (a throwing table decode in the preflight or registry - // read, a non-fc exception from client wiring, a cron add_job - // failure). FC_LOG_AND_DROP is the tree's containment idiom (see - // scan_cycle below) and deliberately rethrows - // boost::interprocess::bad_alloc — chainbase shared-memory - // exhaustion must stay fatal rather than linger as a gate state. - bool completed = false; - try { - attempt_deferred_startup_body(); - completed = true; - } FC_LOG_AND_DROP("underwriter_plugin: deferred startup failed unexpectedly:"); - if (!completed) { - gate_state = underwriter_detail::startup_state::startup_failed; - } + completed = true; + } FC_LOG_AND_DROP("underwriter_plugin: deferred startup failed unexpectedly:"); + if (!completed) { + gate_state = underwriter_detail::startup_state::startup_failed; } // Tripwire for the {@link startup_attempted} derivation: every attempt // that returns normally must have left `waiting_for_sync` — a future // early return in the body before its first gate_state store would - // otherwise strand the node in undiagnosable limbo (gate connection + // otherwise strand the node in undiagnosable limbo (gate subscription // already released, nothing left to re-arm). assert(startup_attempted()); + quit_if_startup_failed_terminally(); + } + + /// The uniform fail-fast policy: a terminal deferred-startup failure shuts + /// the node down instead of leaving a quiet, non-underwriting node behind a + /// running process — an operator daemon that is not performing its duties + /// must be supervisor-visible (it has liveness/slashing consequences), not + /// hidden behind one log line. The specific failure was already stored as a + /// gate state and logged by the attempt; this holds the shutdown decision in + /// ONE place so the policy cannot diverge between the first attempt and the + /// bounded preflight retries. + void quit_if_startup_failed_terminally() { + const auto state = gate_state.load(); + if (!underwriter_detail::is_terminal_failure(state)) { + return; + } + elog("underwriter_plugin: deferred startup failed terminally (state={}) — " + "shutting down node (fail-fast)", magic_enum::enum_name(state)); + app().quit(); } /// The deferred startup body proper — see {@link attempt_deferred_startup} @@ -966,7 +953,7 @@ struct underwriter_plugin::impl { gate_state.load() != underwriter_detail::startup_state::preflight_retrying) { return; } - attempt_deferred_startup(escape_policy::contain_escapes); + attempt_deferred_startup(); }); }); } @@ -2747,6 +2734,17 @@ void underwriter_plugin::plugin_initialize(const variables_map& options) { _impl->cron_plug = &app().get_plugin(); _impl->eth_plug = &app().get_plugin(); _impl->sol_plug = &app().get_plugin(); + + // Operator daemons are designed for read-mode = irreversible: the sync gate + // (controller::is_synced) measures LIB recency and every local table read the + // preflight and scan cycle perform serves the irreversible view. Any other read + // mode would validate/underwrite against state that can still fork out. Together + // with producer_plugin's inverse assert (no producer-name under irreversible + // read-mode), this also makes co-hosting a producer with an operator daemon + // impossible by configuration. + FC_ASSERT(!_impl->enabled || + _impl->chain_plug->chain().get_read_mode() == chain::db_read_mode::IRREVERSIBLE, + "underwriter_plugin requires read-mode = irreversible"); } void underwriter_plugin::plugin_startup() { @@ -2770,59 +2768,59 @@ void underwriter_plugin::plugin_startup() { // registry, authex links) via LOCAL table reads. On a cold-booting // operator node those reads see mid-sync (possibly genesis) state and // would fail spuriously, so the whole startup body (preflight → outpost - // client wiring → cron) is DEFERRED until the node is synced — the LAST - // IRREVERSIBLE block's time within `underwriter_defaults::sync_recency_ms` - // of now (the state the reads actually serve under read-mode = - // irreversible), observed via the controller's `accepted_block` signal. - // Post-arming, the preflight retries within a bounded grace (see - // attempt_deferred_startup) to absorb the LIB boundary race; a genuinely - // incomplete bootstrap still fails loudly once the grace expires — there - // remains no dev escape hatch. + // client wiring → cron) is DEFERRED until the node is synced — + // `controller::is_synced()`: the LAST IRREVERSIBLE block's time within + // `controller::default_sync_recency_ms` of now (the state the reads + // actually serve under read-mode = irreversible). The wake-up is the + // existing `irreversible_block` channel: a LIB advance is the only event + // that can turn the predicate true, and channel deliveries are posted to + // the application executor — main thread, AFTER the triggering block + // fully commits — so the callback may run the startup body directly. + // There is deliberately no already-synced fast path: operator daemons + // boot with genesis-stale LIB in every deployment topology (producer + // co-hosting is impossible by configuration — see the read-mode + // requirement in plugin_initialize), and a node that somehow is synced at + // startup is released by the next LIB advance. That advance is also the + // underwriter's WORK SUPPLY — everything here acts on newly-finalized + // state — so a finality stall that spans startup (the classic lost-wakeup + // case) is one with nothing to underwrite: the first finalized block + // re-arms the gate and creates the first actionable state at the same + // instant. Post-arming, the preflight + // retries within a bounded grace (see attempt_deferred_startup) to absorb + // the LIB boundary race; a genuinely incomplete bootstrap still fails + // loudly once the grace expires and shuts the node down (fail-fast) — + // there remains no dev escape hatch. auto& chain = _impl->chain_plug->chain(); - if (_impl->chain_is_synced()) { - // Synchronous boot path: an escape aborts node startup (fail-fast, - // supervisor-visible) rather than booting with underwriting silently - // disabled. - _impl->run_deferred_startup(impl::escape_policy::abort_on_escape); - return; - } - ilog("underwriter_plugin: waiting for chain sync before preflight " "(head {} is {}s behind now; irreversible state is {}s behind)", chain.head().block_num(), (fc::time_point::now() - chain.head().block_time()).to_seconds(), chain.fork_db_has_root() - ? (fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds() - : -1); - _impl->sync_gate_connection.emplace( - chain.accepted_block().connect([this](const chain::block_signal_params&) { - if (_impl->startup_attempted() || _impl->shutting_down) { - return; - } - if (!_impl->chain_is_synced()) { - return; - } - // Detected sync — disconnect the gate (safe mid-invocation) and POST - // the startup body to the app queue. It must NOT run inline here: - // the accepted_block signal fires mid block-application, where the - // chain API table reads see an incomplete view (observed live: a - // registered operator row read back EMPTY → spurious preflight - // failure). The posted task runs after the block commits, in the - // same main-thread context plugin_startup itself runs in. - _impl->sync_gate_connection->disconnect(); - ilog("underwriter_plugin: chain synced — scheduling deferred startup"); - app().executor().post(appbase::priority::medium, [this]() { - if (_impl->startup_attempted() || _impl->shutting_down) { + ? std::to_string((fc::time_point::now() - chain.fork_db_root().block_time()).to_seconds()) + : "n/a"); + _impl->sync_gate_subscription = + app().get_channel().subscribe( + [impl = _impl.get()](const chain::block_signal_params&) { + if (impl->startup_attempted() || impl->shutting_down || + !impl->chain_plug->chain().is_synced()) { return; } - _impl->run_deferred_startup(impl::escape_policy::contain_escapes); + // One-shot consumption: unsubscribe (safe from within the slot) and + // run the startup body directly — channel deliveries are posted to + // the application executor, so this already runs on the main thread + // AFTER the triggering block committed (the exact context the old + // accepted_block gate had to re-post itself into; running mid + // block-application, table reads see an incomplete view — observed + // live: a registered operator row read back EMPTY → spurious + // preflight failure). + impl->sync_gate_subscription.unsubscribe(); + ilog("underwriter_plugin: chain synced — starting deferred startup"); + impl->run_deferred_startup(); }); - })); } void underwriter_plugin::plugin_shutdown() { _impl->shutting_down = true; - _impl->sync_gate_connection.reset(); if (_impl->preflight_retry_timer) { _impl->preflight_retry_timer->cancel(); } diff --git a/plugins/underwriter_plugin/test/test_underwriter_sync.cpp b/plugins/underwriter_plugin/test/test_underwriter_sync.cpp index ae08632646..4c4a86e4ed 100644 --- a/plugins/underwriter_plugin/test/test_underwriter_sync.cpp +++ b/plugins/underwriter_plugin/test/test_underwriter_sync.cpp @@ -1,66 +1,31 @@ /** * @file test_underwriter_sync.cpp - * @brief Unit tests for the underwriter plugin's sync-gate predicate - * (`sysio::underwriter_detail::block_time_is_recent`) — the pure decision - * behind deferring preflight until the chain plugin reports the node - * synced. + * @brief Unit tests for the underwriter plugin's sync-gate lifecycle surface — + * the `/v1/underwriter/*` gate payloads served until the deferred startup + * completes, and the terminal-failure classification behind the fail-fast + * shutdown. The sync predicate itself is the first-class chain_plugin + * gate; its tests live in `plugins/chain_plugin/test/test_sync_gate.cpp`. */ #include #include #include -using sysio::underwriter_detail::block_time_is_recent; - BOOST_AUTO_TEST_SUITE(underwriter_sync_tests) -/// The reference "now" every case offsets from — an arbitrary fixed instant so -/// the tests are deterministic (no wall clock). -static const fc::time_point reference_now = - fc::time_point{} + fc::days(365 * 50); - -/// The default gate window the plugin runs with. -static const fc::microseconds default_window = - fc::milliseconds(sysio::underwriter_defaults::sync_recency_ms); - -BOOST_AUTO_TEST_CASE(head_at_now_is_synced) { - BOOST_TEST(block_time_is_recent(reference_now, reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_slightly_behind_within_window_is_synced) { - // One block interval behind — the steady-state of a caught-up node. - BOOST_TEST(block_time_is_recent(reference_now - fc::milliseconds(500), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_exactly_at_window_boundary_is_synced) { - BOOST_TEST(block_time_is_recent(reference_now - default_window, - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_just_past_window_is_not_synced) { - BOOST_TEST(!block_time_is_recent(reference_now - default_window - fc::microseconds(1), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(cold_boot_replay_gap_is_not_synced) { - // A node mid-replay: head hours behind wall clock — the exact state whose - // genesis-era table reads used to false-fail the preflight. - BOOST_TEST(!block_time_is_recent(reference_now - fc::hours(2), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(head_ahead_of_now_is_synced) { - // Clock skew / head stamped marginally ahead of the observer's clock must - // never read as "behind". - BOOST_TEST(block_time_is_recent(reference_now + fc::seconds(1), - reference_now, default_window)); -} - -BOOST_AUTO_TEST_CASE(zero_window_requires_head_at_or_past_now) { - BOOST_TEST(block_time_is_recent(reference_now, reference_now, fc::microseconds(0))); - BOOST_TEST(!block_time_is_recent(reference_now - fc::microseconds(1), - reference_now, fc::microseconds(0))); +// ── is_terminal_failure: the fail-fast trigger behind the node shutdown ── + +BOOST_AUTO_TEST_CASE(terminal_failure_states_are_exactly_the_failed_states) { + using sysio::underwriter_detail::is_terminal_failure; + using sysio::underwriter_detail::startup_state; + // Non-failures: the pre-arm wait, the bounded transient retry, and success. + BOOST_TEST(!is_terminal_failure(startup_state::waiting_for_sync)); + BOOST_TEST(!is_terminal_failure(startup_state::preflight_retrying)); + BOOST_TEST(!is_terminal_failure(startup_state::active)); + // Failures: each one must shut the node down (fail-fast). + BOOST_TEST(is_terminal_failure(startup_state::preflight_failed)); + BOOST_TEST(is_terminal_failure(startup_state::wiring_failed)); + BOOST_TEST(is_terminal_failure(startup_state::startup_failed)); } // ── startup_gate_payload: the /v1/underwriter/* body served until the ── diff --git a/unittests/controller_sync_tests.cpp b/unittests/controller_sync_tests.cpp new file mode 100644 index 0000000000..ce036e57a6 --- /dev/null +++ b/unittests/controller_sync_tests.cpp @@ -0,0 +1,97 @@ +/** + * @file controller_sync_tests.cpp + * @brief Unit tests for `controller::is_synced()` — the LIB-recency predicate operator-daemon + * plugins gate their deferred startup on. + * + * The boundary cases drive the testable overload `is_synced(now, recency_window)` with instants + * chosen relative to the tester chain's ACTUAL last-irreversible block time, so every comparison + * exercises the real controller code path deterministically (no wall clock). The two wall-clock + * cases at the end cover the production convenience overload. + */ +#include + +#include +#include + +using namespace sysio::chain; +using namespace sysio::testing; + +BOOST_AUTO_TEST_SUITE(controller_sync_tests) + +/// A produced tester chain plus its last-irreversible block time — the reference instant +/// every boundary case below offsets `now` from. +struct lib_fixture { + tester chain; + fc::time_point lib_time; + /// The default gate window the operator-daemon plugins run with. + fc::microseconds default_window = fc::milliseconds(controller::default_sync_recency_ms); + + lib_fixture() { + chain.produce_blocks(4); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + lib_time = chain.control->fork_db_root().block_time(); + } +}; + +BOOST_FIXTURE_TEST_CASE(lib_at_now_is_synced, lib_fixture) { try { + // The identity point of the window arithmetic: now == LIB time (zero gap). + BOOST_TEST(chain.control->is_synced(lib_time, default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_one_block_interval_behind_is_synced, lib_fixture) { try { + // One block interval behind now — the steady-state of a caught-up node. + BOOST_TEST(chain.control->is_synced(lib_time + fc::milliseconds(500), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_exactly_at_window_boundary_is_synced, lib_fixture) { try { + BOOST_TEST(chain.control->is_synced(lib_time + default_window, default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_just_past_window_is_not_synced, lib_fixture) { try { + BOOST_TEST(!chain.control->is_synced(lib_time + default_window + fc::microseconds(1), + default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(cold_boot_replay_gap_is_not_synced, lib_fixture) { try { + // A node mid-replay: LIB hours behind wall clock — the exact state whose genesis-era + // table reads used to false-fail operator-daemon startups. + BOOST_TEST(!chain.control->is_synced(lib_time + fc::hours(2), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(lib_ahead_of_now_is_synced, lib_fixture) { try { + // Clock skew / LIB stamped marginally ahead of the observer's clock must never read + // as "behind". + BOOST_TEST(chain.control->is_synced(lib_time - fc::seconds(1), default_window)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(zero_window_requires_lib_at_or_past_now, lib_fixture) { try { + BOOST_TEST(chain.control->is_synced(lib_time, fc::microseconds(0))); + BOOST_TEST(!chain.control->is_synced(lib_time + fc::microseconds(1), fc::microseconds(0))); +} FC_LOG_AND_RETHROW() } + +// ── the wall-clock convenience overload the plugins call ── + +BOOST_AUTO_TEST_CASE(stale_chain_is_not_synced) { try { + // The tester genesis timestamp is a fixed instant years in the past, so a freshly + // produced chain's irreversible root time trails wall-clock now by that whole gap — + // the cold-boot state the predicate exists to detect. + tester chain; + chain.produce_blocks(4); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + BOOST_TEST(!chain.control->is_synced()); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_CASE(fresh_chain_is_synced) { try { + // Jump block time from the fixed genesis instant to wall-clock now, then produce a few + // more blocks so the near-now block becomes irreversible. The subsequent block times + // land at/ahead of now (ahead counts as recent — clock-skew rule above), so the + // LIB-recency predicate must hold. + tester chain; + chain.produce_block(); + chain.produce_block(fc::time_point::now() - chain.control->head().block_time()); + chain.produce_blocks(6); + BOOST_REQUIRE(chain.control->fork_db_has_root()); + BOOST_TEST(chain.control->is_synced()); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END()