From 9cf71a9b8bad561f9abdea35408bfbf583f78aa7 Mon Sep 17 00:00:00 2001 From: masarray Date: Tue, 11 Aug 2026 05:49:06 +0700 Subject: [PATCH] Add bounded Smart RCB pre-claim failover --- CMakeLists.txt | 8 ++ REPORTING_PROFILE.md | 10 +- docs/DYNAMIC_RCB_LIVE_TRIAL.md | 44 +++++-- docs/FEATURE_MATRIX.md | 6 +- include/ariec61850/mms/rcb_failover.hpp | 68 ++++++++++ src/mms/rcb_failover.cpp | 82 ++++++++++++ tests/test_rcb_failover.cpp | 159 ++++++++++++++++++++++++ tools/dynamic_rcb_trial.cpp | 72 ++++++++++- 8 files changed, 427 insertions(+), 22 deletions(-) create mode 100644 include/ariec61850/mms/rcb_failover.hpp create mode 100644 src/mms/rcb_failover.cpp create mode 100644 tests/test_rcb_failover.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cdebc85c6..bf5ebc12d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ add_library(ariec61850_core src/mms/tcp_transport.cpp src/mms/live_discovery.cpp src/mms/rcb_selection.cpp + src/mms/rcb_failover.cpp src/sampled_values/pdu_codec.cpp src/sampled_values/frame_codec.cpp src/sampled_values/quality.cpp @@ -216,6 +217,13 @@ if(ARIEC61850_BUILD_TESTS) ariec61850_apply_sanitizers(ariec61850_dynamic_data_set_tests) add_test(NAME ariec61850_dynamic_data_set_tests COMMAND ariec61850_dynamic_data_set_tests) + add_executable(ariec61850_rcb_failover_tests tests/test_rcb_failover.cpp) + target_link_libraries(ariec61850_rcb_failover_tests PRIVATE ARIEC61850::core) + target_compile_features(ariec61850_rcb_failover_tests PRIVATE cxx_std_20) + ariec61850_apply_warnings(ariec61850_rcb_failover_tests) + ariec61850_apply_sanitizers(ariec61850_rcb_failover_tests) + add_test(NAME ariec61850_rcb_failover_tests COMMAND ariec61850_rcb_failover_tests) + add_executable(ariec61850_reporting_tests tests/test_reporting.cpp) target_link_libraries(ariec61850_reporting_tests PRIVATE ARIEC61850::core) target_compile_features(ariec61850_reporting_tests PRIVATE cxx_std_20) diff --git a/REPORTING_PROFILE.md b/REPORTING_PROFILE.md index c2d10d6ca..c52aa3641 100644 --- a/REPORTING_PROFILE.md +++ b/REPORTING_PROFILE.md @@ -14,6 +14,13 @@ optionally reserves an URCB, configures only explicitly requested attributes, en optionally issues GI, receives InformationReports, and cleans up only state touched by the runtime. Lost-association cleanup is recorded rather than assumed. +Before an armed Dynamic RCB lifecycle, the smart pre-claim failover layer runs +bounded repeated Read probes. A candidate that becomes enabled, reserved, or +flaps between probes is excluded from the current command and the pool selector +may rank another candidate. Once any mutation is attempted, automatic switching +is disabled; an uncertain write must be cleaned up and diagnosed on the original +RCB instead of risking a second remote mutation. + ## Phase 4C.1 live-model integration Read-only discovery includes DataSet directory and RCB state evidence in `live-ied-model-v1`. @@ -27,5 +34,6 @@ mode. Runtime report subscription acceptance remains separate from read-only mod - BRCB EntryID resume and replay; - purge policy and buffer-overflow recovery; -- dynamic DataSet lifecycle; +- live proof where a contended preferred RCB is skipped for a second candidate; +- reconnect and automatic resubscribe policy; - long-duration multi-vendor report interoperability. diff --git a/docs/DYNAMIC_RCB_LIVE_TRIAL.md b/docs/DYNAMIC_RCB_LIVE_TRIAL.md index 6289cc3d4..93b55288a 100644 --- a/docs/DYNAMIC_RCB_LIVE_TRIAL.md +++ b/docs/DYNAMIC_RCB_LIVE_TRIAL.md @@ -4,21 +4,27 @@ This harness is for an isolated IEC 61850 laboratory IED/simulator. It intention ## Safety model -Default execution is read-only. It performs live MMS association/discovery, probes RCB state, ranks only empty dynamic slots, and selects scalar ST/MX members. No mutation service is sent unless the exact arm token is present. +Default execution is read-only. It performs live MMS association/discovery, +ranks only empty dynamic slots, and repeats bounded pre-claim Read probes on the +selected candidate. If that candidate is busy or changes state between probes, +it is excluded from the current command and the next safe candidate is ranked. +No mutation service is sent unless the exact arm token is present. An armed run performs one bounded lifecycle with no automatic mutation retry: 1. discover and rank empty dynamic RCB candidates; -2. choose one RCB with empty `DatSet`, `RptEna=false`, and no active reservation; -3. create one uniquely named domain-specific Dynamic DataSet; -4. verify exact member count and order with `GetNamedVariableListAttributes`; -5. bind the selected RCB `DatSet`; -6. reserve URCB when required, enable `RptEna`, and request GI when exposed; -7. use bounded confirmed Read cycles to give asynchronous reports an opportunity to enter the association queue; -8. decode/drain queued reports through the normal reporting runtime; -9. disable the RCB and release only reservations acquired by this runtime; -10. set `DatSet` back to empty; -11. delete only the Dynamic DataSet created by this run. +2. repeatedly probe the selected RCB immediately before mutation; +3. skip busy/flapping candidates, with a hard candidate-attempt bound; +4. choose one stable RCB with empty `DatSet`, `RptEna=false`, and no active reservation; +5. create one uniquely named domain-specific Dynamic DataSet; +6. verify exact member count and order with `GetNamedVariableListAttributes`; +7. bind the selected RCB `DatSet`; +8. reserve URCB when required, enable `RptEna`, and request GI when exposed; +9. use bounded confirmed Read cycles to give asynchronous reports an opportunity to enter the association queue; +10. decode/drain queued reports through the normal reporting runtime; +11. disable the RCB and release only reservations acquired by this runtime; +12. set `DatSet` back to empty; +13. delete only the Dynamic DataSet created by this run. Automatic members are projected from the bounded Logical-Node `GetVariableAccessAttributes` type trees. The planner walks nested FC/DO/DA @@ -32,7 +38,9 @@ Incoming reports accept both `listOfVariable [0]` and the `variableListName [1]` form used by IEDScout, while still decoding only the trailing `listOfAccessResult` as report data. -Cleanup traffic is allowed after a failed mutation, but the user action itself is never retried on another RCB automatically. +Cleanup traffic is allowed after a failed mutation, but the user action itself +is never retried on another RCB automatically. Smart failover applies only to +the read-only pre-claim window, where no remote state has been changed. ## Windows build @@ -54,7 +62,17 @@ powershell.exe -ExecutionPolicy Bypass -File .\scripts\build_dynamic_rcb_trial.p Depending on the generator, the executable can also be directly under `build-dynamic-rcb-trial`. -Expected output includes `RCB selection`, one `Selected` candidate, `SMART_DYNAMIC_RCB_PLAN`, and `READ_ONLY_PLAN_OK`. +Expected output includes `RCB selection`, `PRECLAIM_ATTEMPT`, one stable +candidate, `SMART_DYNAMIC_RCB_PLAN`, and `READ_ONLY_PLAN_OK`. When a candidate +is contended, `RCB_FAILOVER_SKIP` records the reason before another candidate is +ranked. + +The defaults probe up to four candidates, with three Read samples per candidate +and 250 ms between samples. They can be bounded explicitly: + +```powershell +... --max-claim-candidates 4 --preclaim-probes 3 --preclaim-delay-ms 250 --contention-cooldown 60 +``` ## Armed trial diff --git a/docs/FEATURE_MATRIX.md b/docs/FEATURE_MATRIX.md index 7cd162130..3c9eda556 100644 --- a/docs/FEATURE_MATRIX.md +++ b/docs/FEATURE_MATRIX.md @@ -32,14 +32,16 @@ Status terms are intentionally conservative: | Cross-language | Same-IED runtime comparator | Live-proven / mutable | Runtime comparison is informational; C++-only `DataSetBindingStatus` is treated as enrichment rather than a false cross-schema mismatch. | | DataSet | Name inventory | Live-proven | Dynamic inventory may legitimately change between sessions. | | DataSet | Member directory | Live-proven/optional | Explicitly skipped with `--no-datasets`; empty member list then produces evidence warning, not structural failure. | +| DataSet | Dynamic create/verify/delete lifecycle | Live-proven / guarded lab | One bounded lifecycle created a domain-specific DataSet, verified exact member order, used it for reporting, and deleted only the runtime-owned DataSet. | | Reporting | RCB inventory | Live-proven | Current endpoint exposes 286 RCB identities: 8 BRCB and 278 URCB in the C# oracle capture. | | Reporting | Full RCB read-only probe | Live-proven | Current C++ full-probe snapshot classified all 286 RCBs with 0 NotRead and 0 ReadFailed. | | Reporting | Bound/Unbound runtime semantics | Live-proven | One current snapshot observed 6 Bound and 280 Unbound. This is mutable runtime state, not a permanent device capability claim. | | Reporting | Dynamic RCB candidate planner | Live-proven | Read-only ranking selected an empty dynamic BRCB candidate on current endpoint. | | Reporting | Conservative operational availability | Live-proven | Separates dynamic-slot eligibility from populated/static report availability. | | Reporting | Read-only contention pre-claim probe | Live-proven | Three repeated probes on `OCR7SR12CTRL/BI6GGIO1.urcbA01` remained free/stable and returned `StableProceed`; no claim/write was performed. | -| Reporting | RCB claim/reservation/enable | Planned / out of Phase 4C | No Write, reservation, `RptEna` or GI is authorized by live discovery. | -| Reporting | InformationReport decode / subscription runtime | Offline-tested / partial live integration | Core implementation exists; full live reporting acceptance is not claimed. | +| Reporting | Bounded smart pre-claim failover | Offline-tested / stable live probe | Immediately before mutation, repeated read-only probes reject busy/flapping candidates, add them to the command-local exclusion set, and rerank the next candidate. A stable live candidate passed three probes; a live contended switch remains pending. Mutation failures never trigger automatic switching. | +| Reporting | RCB claim/reservation/enable | Live-proven / guarded lab | Explicitly armed Dynamic RCB trial bound and enabled one BRCB, requested GI, then disabled and unbound it with cleanup verified. Live discovery itself remains read-only. | +| Reporting | InformationReport decode / subscription runtime | Live-proven / single profile | The guarded trial received and decoded an InformationReport through the persistent subscription runtime; long-duration and multi-vendor acceptance remain pending. | | GOOSE | PDU / Ethernet encode-decode | Offline-tested | Golden vectors, deterministic tests and fuzz corpus. | | GOOSE | Publisher/subscriber runtime foundations | Offline-tested | Physical multi-vendor process-bus acceptance pending. | | Sampled Values | PDU / Ethernet encode-decode | Offline-tested | Golden vectors, deterministic tests and fuzz corpus. | diff --git a/include/ariec61850/mms/rcb_failover.hpp b/include/ariec61850/mms/rcb_failover.hpp new file mode 100644 index 000000000..f1f8fd662 --- /dev/null +++ b/include/ariec61850/mms/rcb_failover.hpp @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include +#include +#include +#include + +namespace ar::iec61850::mms { + +struct MmsRcbContentionProbeResult; + +enum class MmsRcbPreclaimOutcome { + stable_proceed, + skipped_contended, + attempts_exhausted, +}; + +[[nodiscard]] std::string_view mms_rcb_preclaim_outcome_name( + MmsRcbPreclaimOutcome value) noexcept; + +struct MmsRcbPreclaimAttemptEvidence final { + std::size_t attempt_number{}; + std::string rcb_reference; + MmsRcbPreclaimOutcome outcome{MmsRcbPreclaimOutcome::skipped_contended}; + bool busy{}; + bool flapping{}; + int cooldown_seconds{}; + std::string reason; +}; + +struct MmsRcbPreclaimFailoverSnapshot final { + std::size_t maximum_candidate_attempts{}; + std::vector attempts; + std::vector excluded_rcb_references; + std::string selected_rcb_reference; + bool exhausted{}; + + [[nodiscard]] bool selected() const noexcept { + return !selected_rcb_reference.empty(); + } +}; + +// Bounded policy state for the read-only pre-claim window. A contended RCB is +// excluded from the current command so the pool selector can rank another +// candidate. This tracker intentionally stops being usable after one candidate +// is accepted; mutation failures are not failover signals. +class MmsRcbPreclaimFailoverTracker final { +public: + explicit MmsRcbPreclaimFailoverTracker( + std::size_t maximum_candidate_attempts = 4U); + + [[nodiscard]] bool may_attempt() const noexcept; + [[nodiscard]] const std::vector& excluded_rcb_references() const noexcept { + return snapshot_.excluded_rcb_references; + } + + [[nodiscard]] MmsRcbPreclaimOutcome observe( + const MmsRcbContentionProbeResult& result); + [[nodiscard]] const MmsRcbPreclaimFailoverSnapshot& snapshot() const noexcept { + return snapshot_; + } + +private: + MmsRcbPreclaimFailoverSnapshot snapshot_; +}; + +} // namespace ar::iec61850::mms diff --git a/src/mms/rcb_failover.cpp b/src/mms/rcb_failover.cpp new file mode 100644 index 000000000..8bd779222 --- /dev/null +++ b/src/mms/rcb_failover.cpp @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ariec61850/mms/rcb_failover.hpp" + +#include "ariec61850/mms/rcb_contention.hpp" + +#include +#include +#include + +namespace ar::iec61850::mms { + +std::string_view mms_rcb_preclaim_outcome_name( + const MmsRcbPreclaimOutcome value) noexcept { + switch (value) { + case MmsRcbPreclaimOutcome::stable_proceed: return "StableProceed"; + case MmsRcbPreclaimOutcome::skipped_contended: return "SkippedContended"; + case MmsRcbPreclaimOutcome::attempts_exhausted: return "AttemptsExhausted"; + } + return "Unknown"; +} + +MmsRcbPreclaimFailoverTracker::MmsRcbPreclaimFailoverTracker( + const std::size_t maximum_candidate_attempts) { + if (maximum_candidate_attempts == 0U) { + throw std::invalid_argument( + "RCB pre-claim failover requires at least one candidate attempt."); + } + snapshot_.maximum_candidate_attempts = maximum_candidate_attempts; +} + +bool MmsRcbPreclaimFailoverTracker::may_attempt() const noexcept { + return !snapshot_.selected() && !snapshot_.exhausted && + snapshot_.attempts.size() < snapshot_.maximum_candidate_attempts; +} + +MmsRcbPreclaimOutcome MmsRcbPreclaimFailoverTracker::observe( + const MmsRcbContentionProbeResult& result) { + if (!may_attempt()) { + throw std::logic_error( + "RCB pre-claim failover is already selected or exhausted."); + } + if (result.rcb_reference.empty()) { + throw std::invalid_argument( + "RCB pre-claim probe result requires a candidate reference."); + } + const auto duplicate = std::find( + snapshot_.excluded_rcb_references.begin(), + snapshot_.excluded_rcb_references.end(), + result.rcb_reference); + if (duplicate != snapshot_.excluded_rcb_references.end()) { + throw std::logic_error( + "RCB pre-claim failover cannot probe an excluded candidate twice."); + } + + MmsRcbPreclaimAttemptEvidence attempt; + attempt.attempt_number = snapshot_.attempts.size() + 1U; + attempt.rcb_reference = result.rcb_reference; + attempt.busy = result.is_busy_at_probe; + attempt.flapping = result.is_flapping; + attempt.cooldown_seconds = result.cooldown_seconds; + attempt.reason = result.reason; + + if (!result.is_contended) { + attempt.outcome = MmsRcbPreclaimOutcome::stable_proceed; + snapshot_.selected_rcb_reference = result.rcb_reference; + snapshot_.attempts.push_back(std::move(attempt)); + return MmsRcbPreclaimOutcome::stable_proceed; + } + + snapshot_.excluded_rcb_references.push_back(result.rcb_reference); + const bool exhausted = + snapshot_.attempts.size() + 1U >= snapshot_.maximum_candidate_attempts; + attempt.outcome = exhausted + ? MmsRcbPreclaimOutcome::attempts_exhausted + : MmsRcbPreclaimOutcome::skipped_contended; + snapshot_.exhausted = exhausted; + snapshot_.attempts.push_back(std::move(attempt)); + return snapshot_.attempts.back().outcome; +} + +} // namespace ar::iec61850::mms diff --git a/tests/test_rcb_failover.cpp b/tests/test_rcb_failover.cpp new file mode 100644 index 000000000..27bf0e77d --- /dev/null +++ b/tests/test_rcb_failover.cpp @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ariec61850/mms/rcb_contention.hpp" +#include "ariec61850/mms/rcb_failover.hpp" +#include "ariec61850/mms/rcb_selection.hpp" + +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ar::iec61850; + +#define CHECK(condition) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error( \ + std::string{"CHECK failed: "} + #condition + \ + " at " + __FILE__ + ":" + std::to_string(__LINE__)); \ + } \ + } while (false) + +[[nodiscard]] mms::MmsRcbContentionProbeResult probe( + std::string reference, + const bool busy, + const bool flapping = false) { + mms::MmsRcbContentionProbeResult result; + result.rcb_reference = std::move(reference); + result.is_busy_at_probe = busy; + result.is_flapping = flapping; + result.is_contended = busy || flapping; + result.cooldown_seconds = result.is_contended ? 60 : 0; + result.reason = result.is_contended ? "contended" : "stable"; + return result; +} + +[[nodiscard]] mms::MmsReportControlCandidate candidate(std::string name) { + mms::MmsReportControlCandidate value; + value.domain = "LD0"; + value.logical_node = "LLN0"; + value.functional_constraint = "BR"; + value.name = std::move(name); + value.reference = "LD0/LLN0.BR." + value.name; + value.buffered = true; + value.attributes = {"DatSet", "RptEna", "ResvTms"}; + return value; +} + +void add_free_state( + mms::MmsLiveDiscoveryResult& discovery, + const mms::MmsReportControlCandidate& value) { + mms::MmsReportControlEvidence evidence; + evidence.candidate = value; + evidence.requested_attributes = {"DatSet", "RptEna", "ResvTms"}; + mms::MmsReportControlState state; + state.candidate = value; + state.report_enabled = false; + state.reservation_time_seconds = 0U; + evidence.state = std::move(state); + discovery.report_controls.push_back(std::move(evidence)); +} + +void stable_candidate_is_accepted_once() { + mms::MmsRcbPreclaimFailoverTracker tracker{3U}; + CHECK(tracker.may_attempt()); + CHECK(tracker.observe(probe("LD0/LLN0.BR.a", false)) == + mms::MmsRcbPreclaimOutcome::stable_proceed); + CHECK(!tracker.may_attempt()); + CHECK(tracker.snapshot().selected_rcb_reference == "LD0/LLN0.BR.a"); + CHECK(tracker.snapshot().excluded_rcb_references.empty()); +} + +void contended_candidate_is_excluded_before_fallback() { + mms::MmsRcbPreclaimFailoverTracker tracker{3U}; + CHECK(tracker.observe(probe("LD0/LLN0.BR.a", true)) == + mms::MmsRcbPreclaimOutcome::skipped_contended); + CHECK(tracker.may_attempt()); + CHECK(tracker.excluded_rcb_references() == + std::vector{"LD0/LLN0.BR.a"}); + CHECK(tracker.observe(probe("LD0/LLN0.BR.b", false)) == + mms::MmsRcbPreclaimOutcome::stable_proceed); + CHECK(tracker.snapshot().attempts.size() == 2U); + CHECK(tracker.snapshot().selected_rcb_reference == "LD0/LLN0.BR.b"); +} + +void maximum_attempts_are_hard_bounded() { + mms::MmsRcbPreclaimFailoverTracker tracker{2U}; + CHECK(tracker.observe(probe("LD0/LLN0.BR.a", false, true)) == + mms::MmsRcbPreclaimOutcome::skipped_contended); + CHECK(tracker.observe(probe("LD0/LLN0.BR.b", true)) == + mms::MmsRcbPreclaimOutcome::attempts_exhausted); + CHECK(tracker.snapshot().exhausted); + CHECK(!tracker.may_attempt()); +} + +void excluded_preferred_candidate_reranks_to_next_slot() { + mms::MmsLiveDiscoveryResult discovery; + const auto preferred = candidate("a"); + const auto fallback = candidate("b"); + discovery.report_inventory.report_controls = {preferred, fallback}; + add_free_state(discovery, preferred); + add_free_state(discovery, fallback); + + mms::MmsRcbDynamicSelectionOptions options; + options.preferred_rcb_reference = preferred.reference; + auto selection = mms::MmsRcbPoolSelector::build_dynamic_selection( + discovery, options); + CHECK(selection.selected_rcb_reference == preferred.reference); + + mms::MmsRcbPreclaimFailoverTracker tracker{3U}; + CHECK(tracker.observe(probe(preferred.reference, true)) == + mms::MmsRcbPreclaimOutcome::skipped_contended); + options.excluded_rcb_references = tracker.excluded_rcb_references(); + selection = mms::MmsRcbPoolSelector::build_dynamic_selection( + discovery, options); + CHECK(selection.selected_rcb_reference == fallback.reference); + CHECK(selection.fallback_used); +} + +void invalid_and_duplicate_observations_are_rejected() { + bool zero_rejected = false; + try { + static_cast(mms::MmsRcbPreclaimFailoverTracker{0U}); + } catch (const std::invalid_argument&) { + zero_rejected = true; + } + CHECK(zero_rejected); + + mms::MmsRcbPreclaimFailoverTracker tracker{3U}; + static_cast(tracker.observe(probe("LD0/LLN0.BR.a", true))); + bool duplicate_rejected = false; + try { + static_cast(tracker.observe(probe("LD0/LLN0.BR.a", true))); + } catch (const std::logic_error&) { + duplicate_rejected = true; + } + CHECK(duplicate_rejected); +} + +} // namespace + +int main() { + try { + stable_candidate_is_accepted_once(); + contended_candidate_is_excluded_before_fallback(); + maximum_attempts_are_hard_bounded(); + excluded_preferred_candidate_reranks_to_next_slot(); + invalid_and_duplicate_observations_are_rejected(); + std::cout << "RCB failover tests passed\n"; + return 0; + } catch (const std::exception& exception) { + std::cerr << exception.what() << '\n'; + return 1; + } +} diff --git a/tools/dynamic_rcb_trial.cpp b/tools/dynamic_rcb_trial.cpp index 72d8a110b..a78f1cc59 100644 --- a/tools/dynamic_rcb_trial.cpp +++ b/tools/dynamic_rcb_trial.cpp @@ -3,6 +3,8 @@ #include "ariec61850/mms/dynamic_data_set.hpp" #include "ariec61850/mms/dynamic_report_planner.hpp" #include "ariec61850/mms/live_discovery.hpp" +#include "ariec61850/mms/rcb_contention.hpp" +#include "ariec61850/mms/rcb_failover.hpp" #include "ariec61850/mms/rcb_selection.hpp" #include "ariec61850/mms/report_subscription_runtime.hpp" #include "ariec61850/mms/services.hpp" @@ -40,6 +42,10 @@ struct CliOptions final { std::size_t auto_member_count{4U}; std::size_t maximum_type_probes{2'048U}; std::size_t maximum_rcb_probes{4'096U}; + std::size_t maximum_claim_candidates{4U}; + std::size_t preclaim_probe_count{3U}; + std::chrono::milliseconds preclaim_probe_delay{250}; + int contention_cooldown_seconds{60}; std::size_t probe_cycles{3U}; std::chrono::milliseconds probe_delay{150}; bool allow_urcb_fallback{true}; @@ -77,6 +83,10 @@ void print_usage() { << " --auto-members N Auto-pick N scalar ST/MX members (default 4).\n" << " --max-types N Bound GVAA probes used for auto-members (default 2048).\n" << " --max-rcb N Bound RCB probes (default 4096).\n" + << " --max-claim-candidates N Bound pre-claim candidates (default 4).\n" + << " --preclaim-probes N Read-only probes per candidate (default 3).\n" + << " --preclaim-delay-ms N Delay between pre-claim probes (default 250).\n" + << " --contention-cooldown N Recorded skip cooldown seconds (default 60).\n" << " --probe-cycles N Confirmed read cycles after GI (default 3).\n" << " --probe-delay-ms N Delay between confirmation reads (default 150).\n" << " --timeout-ms N Connect/request timeout (default 5000).\n" @@ -85,7 +95,8 @@ void print_usage() { << " Enable mutation: Define DataSet -> bind RCB -> enable/GI\n" << " -> observe -> disable -> unbind -> Delete DataSet.\n" << " -h, --help Show this help.\n\n" - << "This harness never auto-retries a failed mutation and never disables a busy RCB.\n"; + << "Busy/flapping candidates are skipped before mutation. This harness never\n" + << "auto-retries a failed mutation and never disables a busy RCB.\n"; } [[nodiscard]] CliOptions parse_cli(int argc, char** argv) { @@ -104,7 +115,9 @@ void print_usage() { else if (option == "--rcb-ld" || option == "--preferred-rcb" || option == "--dataset-ref" || option == "--member" || option == "--auto-members" || option == "--max-types" || - option == "--max-rcb" || option == "--probe-cycles" || + option == "--max-rcb" || option == "--max-claim-candidates" || + option == "--preclaim-probes" || option == "--preclaim-delay-ms" || + option == "--contention-cooldown" || option == "--probe-cycles" || option == "--probe-delay-ms" || option == "--timeout-ms" || option == "--arm") { if (index >= argc) throw std::invalid_argument(option + " requires a value."); @@ -116,6 +129,17 @@ void print_usage() { else if (option == "--auto-members") options.auto_member_count = parse_positive(option, value); else if (option == "--max-types") options.maximum_type_probes = parse_positive(option, value); else if (option == "--max-rcb") options.maximum_rcb_probes = parse_positive(option, value); + else if (option == "--max-claim-candidates") options.maximum_claim_candidates = parse_positive(option, value); + else if (option == "--preclaim-probes") options.preclaim_probe_count = parse_positive(option, value); + else if (option == "--preclaim-delay-ms") { + options.preclaim_probe_delay = std::chrono::milliseconds{static_cast(parse_positive(option, value))}; + } else if (option == "--contention-cooldown") { + const auto seconds = parse_positive(option, value); + if (seconds > static_cast(std::numeric_limits::max())) { + throw std::invalid_argument(option + " exceeds the supported integer range."); + } + options.contention_cooldown_seconds = static_cast(seconds); + } else if (option == "--probe-cycles") options.probe_cycles = parse_positive(option, value); else if (option == "--probe-delay-ms") { options.probe_delay = std::chrono::milliseconds{static_cast(parse_positive(option, value))}; @@ -281,12 +305,46 @@ int main(int argc, char** argv) { selection_options.preferred_rcb_reference = cli.preferred_rcb_reference; selection_options.allow_urcb_fallback = cli.allow_urcb_fallback; selection_options.allow_polling_fallback = false; - const auto selection = mms::MmsRcbPoolSelector::build_dynamic_selection(discovery, selection_options); - print_candidates(selection); - const auto* candidate = mms::MmsRcbPoolSelector::select_report_control(discovery, selection); + mms::MmsRcbPreclaimFailoverTracker failover{cli.maximum_claim_candidates}; + mms::MmsRcbContentionProbeClient contention_probe{session.association()}; + const mms::MmsReportControlCandidate* candidate = nullptr; + while (failover.may_attempt()) { + selection_options.excluded_rcb_references = + failover.excluded_rcb_references(); + const auto selection = mms::MmsRcbPoolSelector::build_dynamic_selection( + discovery, selection_options); + print_candidates(selection); + candidate = mms::MmsRcbPoolSelector::select_report_control( + discovery, selection); + if (candidate == nullptr) { + break; + } + + mms::MmsRcbContentionProbeOptions probe_options; + probe_options.probe_count = cli.preclaim_probe_count; + probe_options.probe_delay = cli.preclaim_probe_delay; + probe_options.cooldown_seconds = cli.contention_cooldown_seconds; + const auto probe = contention_probe.probe(*candidate, probe_options); + const auto outcome = failover.observe(probe); + const auto& attempt = failover.snapshot().attempts.back(); + std::cout << "PRECLAIM_ATTEMPT attempt=" << attempt.attempt_number + << " rcb=" << attempt.rcb_reference + << " outcome=" << mms::mms_rcb_preclaim_outcome_name(outcome) + << " busy=" << (attempt.busy ? "true" : "false") + << " flapping=" << (attempt.flapping ? "true" : "false") + << " cooldownSeconds=" << attempt.cooldown_seconds << '\n'; + if (outcome == mms::MmsRcbPreclaimOutcome::stable_proceed) { + break; + } + std::cout << "RCB_FAILOVER_SKIP rcb=" << attempt.rcb_reference + << " reason=" << attempt.reason << '\n'; + candidate = nullptr; + } if (candidate == nullptr) { session.disconnect(); - std::cerr << "BLOCKED: no safe empty dynamic RCB slot was found. No mutation sent.\n"; + std::cerr << "BLOCKED: no stable empty dynamic RCB slot remained after " + << failover.snapshot().attempts.size() + << " bounded pre-claim attempt(s). No mutation sent.\n"; return 3; } if (!contains_attribute(*candidate, "DatSet") || !contains_attribute(*candidate, "RptEna")) { @@ -325,6 +383,8 @@ int main(int argc, char** argv) { std::cout << "SMART_DYNAMIC_RCB_PLAN\n" << " endpoint=" << cli.endpoint.host << ':' << cli.endpoint.port << '\n' << " rcb=" << candidate->reference << " mode=" << candidate->mode() << '\n' + << " preclaimAttempts=" << failover.snapshot().attempts.size() + << " skipped=" << failover.snapshot().excluded_rcb_references.size() << '\n' << " dataset=" << data_set_reference << '\n' << " members=" << members.size() << '\n'; if (auto_selection) {