From 9da6fe7ad5a0762785e63462eea3ab615b7a894c Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:10:01 +0800 Subject: [PATCH 1/4] fix #413: Follower/Learner silently ignore stale Leader-only RoleEvents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After migrating 6 lifecycle events from bounded event_tx (P4) to unbounded role_tx (P2), BecomeFollower and these events now share the same channel. The ordering is a race: Follower receiving LogPurgeCompleted / PromoteReadyLearners / MembershipApplied after step-down is normal in-flight behavior, not a protocol violation. - follower_state: override 3 methods → Ok(()) instead of default RoleViolation - learner_state: override 2 methods → Ok(()) - raft.rs: add is_fatal() guard to StepDownSelfRemoved and LogPurgeCompleted dispatch arms (symmetry with other 4 arms) - role_state: fix hardcoded current_role "Learner" → "non-Leader" in default handle_log_purge_completed error message - Add dispatch-layer tests for the 6 migrated RoleEvent variants - Add LIFECYCLE_EVENT_MIGRATION_ANALYSIS.md for protocol correctness review Candidate retains RoleViolation — it cannot receive stale Leader events because role_tx is fully drained between Leader→Follower and Follower→Candidate. --- d-engine-core/benches/leader_state_bench.rs | 5 +- .../commit_handler/default_commit_handler.rs | 16 +- .../default_commit_handler_test.rs | 65 +- d-engine-core/src/commit_handler/mod.rs | 15 +- d-engine-core/src/event.rs | 54 +- d-engine-core/src/raft.rs | 79 +- .../src/raft_role/candidate_state.rs | 84 +-- .../src/raft_role/candidate_state_test.rs | 36 +- d-engine-core/src/raft_role/follower_state.rs | 240 +++--- .../src/raft_role/follower_state_test.rs | 268 +++---- d-engine-core/src/raft_role/leader_state.rs | 689 +++++++++--------- .../leader_state_test/event_handling_test.rs | 4 +- .../membership_change_test.rs | 21 +- .../leader_state_test/snapshot_test.rs | 78 +- d-engine-core/src/raft_role/learner_state.rs | 281 +++---- .../src/raft_role/learner_state_test.rs | 196 ++--- d-engine-core/src/raft_role/mod.rs | 69 ++ d-engine-core/src/raft_role/role_state.rs | 130 +++- d-engine-core/src/raft_test/mod.rs | 19 + .../src/raft_test/raft_comprehensive_tests.rs | 161 ++++ d-engine-proto/proto/server/storage.proto | 11 +- .../src/generated/d_engine.server.storage.rs | 33 +- d-engine-server/src/node/builder.rs | 4 +- 23 files changed, 1429 insertions(+), 1129 deletions(-) diff --git a/d-engine-core/benches/leader_state_bench.rs b/d-engine-core/benches/leader_state_bench.rs index 94c87c4f..d690b869 100644 --- a/d-engine-core/benches/leader_state_bench.rs +++ b/d-engine-core/benches/leader_state_bench.rs @@ -112,6 +112,7 @@ use std::time::Duration; use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; use d_engine_core::leader_state::{LeaderState, PendingPromotion}; +use d_engine_core::role_state::RaftRoleState; use d_engine_core::{ MockElectionCore, MockMembership, MockPurgeExecutor, MockRaftLog, MockReplicationCore, MockStateMachine, MockStateMachineHandler, MockTransport, MockTypeConfig, RaftContext, @@ -286,7 +287,7 @@ fn bench_process_promotions_2_nodes(c: &mut Criterion) { black_box( fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await, ) }) @@ -319,7 +320,7 @@ fn bench_batch_promotion_scaling(c: &mut Criterion) { black_box( fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await, ) }) diff --git a/d-engine-core/src/commit_handler/default_commit_handler.rs b/d-engine-core/src/commit_handler/default_commit_handler.rs index eae504e5..ca4dfb4f 100644 --- a/d-engine-core/src/commit_handler/default_commit_handler.rs +++ b/d-engine-core/src/commit_handler/default_commit_handler.rs @@ -13,9 +13,9 @@ use tracing::warn; use super::CommitHandler; use crate::Membership; use crate::NewCommitData; -use crate::RaftEvent; use crate::RaftLog; use crate::Result; +use crate::RoleEvent; use crate::StateMachineHandler; use crate::TypeConfig; use crate::alias::MOF; @@ -28,7 +28,7 @@ pub struct CommitHandlerDependencies { pub state_machine_handler: Arc>, pub raft_log: Arc>, pub membership: Arc>, - pub event_tx: mpsc::Sender, + pub role_tx: mpsc::UnboundedSender, pub sm_apply_tx: mpsc::UnboundedSender>, pub shutdown_signal: watch::Receiver<()>, pub max_batch_size: usize, @@ -47,7 +47,7 @@ where new_commit_rx: Option>, membership: Arc>, - event_tx: mpsc::Sender, // Cloned from Raft + role_tx: mpsc::UnboundedSender, // Cloned from Raft sm_apply_tx: mpsc::UnboundedSender>, // Send entries to SM Worker // Shutdown signal @@ -130,7 +130,7 @@ where raft_log: deps.raft_log, membership: deps.membership, new_commit_rx: Some(new_commit_rx), - event_tx: deps.event_tx, + role_tx: deps.role_tx, sm_apply_tx: deps.sm_apply_tx, shutdown_signal: deps.shutdown_signal, max_batch_size: deps.max_batch_size, @@ -254,7 +254,7 @@ where // 2.5. Notify leader to refresh cluster metadata cache // This must happen AFTER membership is applied - if let Err(e) = self.event_tx.send(RaftEvent::MembershipApplied).await { + if let Err(e) = self.role_tx.send(RoleEvent::MembershipApplied) { warn!("Failed to send MembershipApplied event: {:?}", e); } @@ -265,7 +265,7 @@ where self.my_id, entry.index ); // Signal step down - error is non-fatal as removal is already committed - if let Err(e) = self.event_tx.send(RaftEvent::StepDownSelfRemoved).await { + if let Err(e) = self.role_tx.send(RoleEvent::StepDownSelfRemoved) { error!( "[{}] Failed to send StepDownSelfRemoved event: {:?}", self.my_id, e @@ -299,3 +299,7 @@ where Ok(()) } } + +#[cfg(test)] +#[path = "default_commit_handler_test.rs"] +mod default_commit_handler_test; diff --git a/d-engine-core/src/commit_handler/default_commit_handler_test.rs b/d-engine-core/src/commit_handler/default_commit_handler_test.rs index eaa7c247..077e0b61 100644 --- a/d-engine-core/src/commit_handler/default_commit_handler_test.rs +++ b/d-engine-core/src/commit_handler/default_commit_handler_test.rs @@ -1,11 +1,10 @@ -use std::sync::Arc; -use std::time::Duration; - use bytes::Bytes; use d_engine_proto::common::Entry; use d_engine_proto::common::EntryPayload; use d_engine_proto::common::NodeRole::Leader; use d_engine_proto::common::membership_change::Change; +use std::sync::Arc; +use std::time::Duration; use tokio::sync::mpsc::{self}; use tokio::sync::watch; use tokio::task::JoinHandle; @@ -20,8 +19,8 @@ use crate::MockRaftLog; use crate::MockStateMachineHandler; use crate::MockTypeConfig; use crate::NewCommitData; -use crate::RaftEvent; use crate::Result; +use crate::RoleEvent; const TEST_TERM: u64 = 1; @@ -69,8 +68,8 @@ pub struct TestHarness { mock_membership: Arc>, commit_tx: mpsc::UnboundedSender, commit_rx: Option>, - event_tx: mpsc::Sender, - event_rx: mpsc::Receiver, + role_tx: mpsc::UnboundedSender, + role_rx: mpsc::UnboundedReceiver, shutdown_tx: watch::Sender<()>, shutdown_rx: Option>, handle: Option>, @@ -91,7 +90,7 @@ where { let (commit_tx, commit_rx) = mpsc::unbounded_channel(); let (shutdown_tx, shutdown_rx) = watch::channel(()); - let (event_tx, event_rx) = mpsc::channel(1000); // Large capacity to prevent blocking in tests + let (role_tx, role_rx) = mpsc::unbounded_channel(); // Large capacity to prevent blocking in tests // Mock state machine let mut mock_smh = MockStateMachineHandler::new(); @@ -135,8 +134,8 @@ where mock_membership: Arc::new(mock_membership), commit_rx: Some(commit_rx), commit_tx, - event_tx, - event_rx, + role_tx, + role_rx, shutdown_tx, shutdown_rx: Some(shutdown_rx), handle: None, @@ -151,7 +150,7 @@ impl TestHarness { state_machine_handler: self.mock_smh.clone(), raft_log: self.mock_log.clone(), membership: self.mock_membership.clone(), - event_tx: self.event_tx.clone(), + role_tx: self.role_tx.clone(), sm_apply_tx, shutdown_signal: self.shutdown_rx.take().unwrap(), max_batch_size: 10, @@ -186,7 +185,7 @@ impl TestHarness { state_machine_handler: self.mock_smh.clone(), raft_log: self.mock_log.clone(), membership: self.mock_membership.clone(), - event_tx: self.event_tx.clone(), + role_tx: self.role_tx.clone(), sm_apply_tx, shutdown_signal: self.shutdown_rx.take().unwrap(), max_batch_size: 10, @@ -225,7 +224,7 @@ impl TestHarness { } async fn expect_snapshot_trigger(&mut self) -> bool { - match time::timeout(Duration::from_millis(50), self.event_rx.recv()).await { + match time::timeout(Duration::from_millis(50), self.role_rx.recv()).await { Ok(Some(_)) => true, // Event received normally Ok(None) => false, // Channel closed Err(_) => false, // Timeout @@ -651,7 +650,7 @@ mod process_batch_test { // snapshot_condition: Option, // ) -> ( // DefaultCommitHandler, - // mpsc::Receiver, + // mpsc::Receiver, // watch::Sender<()>, // ) where // F: Fn() -> bool + 'static + Send + Sync, @@ -911,7 +910,7 @@ mod process_batch_test { /// Test 1: MembershipApplied event MUST be sent after successful config change /// /// Verifies that when apply_config_change() succeeds, the commit handler - /// sends RaftEvent::MembershipApplied to notify Leader to refresh cache. + /// sends RoleEvent::MembershipApplied to notify Leader to refresh cache. /// /// Related: Bug fix #209 - cluster metadata cache timing issue #[tokio::test] @@ -940,8 +939,8 @@ mod process_batch_test { assert!(result.is_ok(), "Config change should succeed"); // Verify MembershipApplied event was sent - match tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await { - Ok(Some(RaftEvent::MembershipApplied)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { + Ok(Some(RoleEvent::MembershipApplied)) => { // Success - event received } Ok(Some(other)) => panic!("Expected MembershipApplied, got {other:?}"), @@ -994,8 +993,8 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); // Verify event received - match tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await { - Ok(Some(RaftEvent::MembershipApplied)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { + Ok(Some(RoleEvent::MembershipApplied)) => { // Record event reception apply_order.lock().push("MembershipApplied_event"); } @@ -1043,7 +1042,7 @@ mod process_batch_test { assert!(result.is_err(), "Config change should fail"); // Verify NO event was sent - match tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await { + match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { Err(_) => { // Timeout is expected - no event sent } @@ -1077,8 +1076,8 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); assert!( matches!( - tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await, - Ok(Some(RaftEvent::MembershipApplied)) + tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await, + Ok(Some(RoleEvent::MembershipApplied)) ), "AddNode should send MembershipApplied" ); @@ -1094,8 +1093,8 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); assert!( matches!( - tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await, - Ok(Some(RaftEvent::MembershipApplied)) + tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await, + Ok(Some(RoleEvent::MembershipApplied)) ), "RemoveNode should send MembershipApplied" ); @@ -1114,8 +1113,8 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); assert!( matches!( - tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await, - Ok(Some(RaftEvent::MembershipApplied)) + tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await, + Ok(Some(RoleEvent::MembershipApplied)) ), "PromoteLearner should send MembershipApplied" ); @@ -1147,8 +1146,8 @@ mod process_batch_test { // Replace the receiver with a dummy so the original event_tx sends will fail. // Swap old_rx out of the harness so harness is still usable, then drop old_rx. - let (_dummy_tx, dummy_rx) = mpsc::channel::(1); - let old_rx = std::mem::replace(&mut harness.event_rx, dummy_rx); + let (_dummy_tx, dummy_rx) = mpsc::unbounded_channel::(); + let old_rx = std::mem::replace(&mut harness.role_rx, dummy_rx); drop(old_rx); // Now harness.event_tx's peer receiver is gone → sends fail. let result = harness.process_batch_handler().await; @@ -1182,8 +1181,8 @@ mod process_batch_test { ); // Replace receiver with a dummy so both sends fail. - let (_dummy_tx, dummy_rx) = mpsc::channel::(1); - let old_rx = std::mem::replace(&mut harness.event_rx, dummy_rx); + let (_dummy_tx, dummy_rx) = mpsc::unbounded_channel::(); + let old_rx = std::mem::replace(&mut harness.role_rx, dummy_rx); drop(old_rx); let result = harness.process_batch_handler().await; @@ -1226,8 +1225,8 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); // First event should be MembershipApplied - match tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await { - Ok(Some(RaftEvent::MembershipApplied)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { + Ok(Some(RoleEvent::MembershipApplied)) => { // Correct first event } Ok(Some(other)) => panic!("Expected MembershipApplied first, got {other:?}"), @@ -1235,8 +1234,8 @@ mod process_batch_test { } // Second event should be StepDownSelfRemoved - match tokio::time::timeout(Duration::from_millis(50), harness.event_rx.recv()).await { - Ok(Some(RaftEvent::StepDownSelfRemoved)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { + Ok(Some(RoleEvent::StepDownSelfRemoved)) => { // Correct second event } Ok(Some(other)) => panic!("Expected StepDownSelfRemoved second, got {other:?}"), diff --git a/d-engine-core/src/commit_handler/mod.rs b/d-engine-core/src/commit_handler/mod.rs index 81f70dd5..f7bc1f0a 100644 --- a/d-engine-core/src/commit_handler/mod.rs +++ b/d-engine-core/src/commit_handler/mod.rs @@ -50,20 +50,11 @@ //! - Developers must ensure that the state machine implementation is idempotent to handle potential //! retries or re-applications of log entries. mod default_commit_handler; - pub use default_commit_handler::*; -#[cfg(test)] -mod default_commit_handler_test; - -use async_trait::async_trait; -#[cfg(any(test, feature = "__test_support"))] -use mockall::automock; - -use crate::Result; -#[cfg_attr(any(test, feature = "__test_support"), automock)] +#[cfg_attr(any(test, feature = "__test_support"), mockall::automock)] #[doc(hidden)] -#[async_trait] +#[async_trait::async_trait] pub trait CommitHandler: Send + Sync + 'static { - async fn run(&mut self) -> Result<()>; + async fn run(&mut self) -> crate::Result<()>; } diff --git a/d-engine-core/src/event.rs b/d-engine-core/src/event.rs index 31d12e33..5cfb8253 100644 --- a/d-engine-core/src/event.rs +++ b/d-engine-core/src/event.rs @@ -105,6 +105,30 @@ pub enum RoleEvent { results: Vec, }, + /// Trigger state machine snapshot creation — routed via P2 (unbounded role_tx) to + /// prevent deadlock when P4 event_tx is saturated by inbound RPCs. + CreateSnapshotEvent, + + /// Snapshot file is ready; contains metadata and final path. + /// Leader: schedules log purge. Follower/Learner: updates snapshot state. + SnapshotCreated(Result<(SnapshotMetadata, std::path::PathBuf)>), + + /// Node removed itself from cluster membership + /// Leader must step down immediately after self-removal per Raft protocol + StepDownSelfRemoved, + + /// Membership change has been applied to state + /// Leader should refresh cluster metadata cache + MembershipApplied, + + /// Check pending learners for promotion eligibility after a membership change or heartbeat. + /// Re-queued via role_tx until all pending promotions are resolved. + PromoteReadyLearners, + + /// Log entries up to `LogId` have been purged from storage after snapshot creation. + /// Leader updates `last_purged_index` to track the safe purge boundary. + LogPurgeCompleted(LogId), + /// Bidi replication stream to a follower broke (network disconnect or error). /// Emitted by the replication worker's recv task when it gets an Err from the stream. /// Raft loop resets `next_index[peer] = match_index[peer] + 1` so that the next @@ -171,23 +195,6 @@ pub enum RaftEvent { MaybeCloneOneshotSender>, ), - CreateSnapshotEvent, - - LogPurgeCompleted(LogId), - - SnapshotCreated(Result<(SnapshotMetadata, std::path::PathBuf)>), - - // Lightweight promotion trigger - PromoteReadyLearners, - - /// Node removed itself from cluster membership - /// Leader must step down immediately after self-removal per Raft protocol - StepDownSelfRemoved, - - /// Membership change has been applied to state - /// Leader should refresh cluster metadata cache - MembershipApplied, - /// State machine apply failed - node must shutdown. /// Conservative: treat all SM errors as fatal (future: distinguish fatal vs application errors). FatalError { @@ -251,19 +258,6 @@ pub(crate) fn raft_event_to_test_event(event: &RaftEvent) -> TestEvent { RaftEvent::StreamSnapshot(_, _, _) => TestEvent::StreamSnapshot, RaftEvent::JoinCluster(req, _) => TestEvent::JoinCluster(req.clone()), RaftEvent::DiscoverLeader(req, _) => TestEvent::DiscoverLeader(req.clone()), - RaftEvent::CreateSnapshotEvent => TestEvent::CreateSnapshotEvent, - RaftEvent::SnapshotCreated(_result) => TestEvent::SnapshotCreated, - RaftEvent::LogPurgeCompleted(id) => TestEvent::LogPurgeCompleted(*id), - RaftEvent::PromoteReadyLearners => TestEvent::PromoteReadyLearners, - RaftEvent::StepDownSelfRemoved => { - // StepDownSelfRemoved is handled at Raft level, not converted to TestEvent - // This is a control flow event, not a user-facing event - TestEvent::CreateSnapshotEvent // Placeholder - this event won't be emitted to tests - } - RaftEvent::MembershipApplied => { - // MembershipApplied is internal event for cache refresh - TestEvent::CreateSnapshotEvent // Placeholder - } RaftEvent::FatalError { source, error } => TestEvent::FatalError { source: source.clone(), error: error.clone(), diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index ad80b95e..2e560ff2 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -540,9 +540,8 @@ where if let Err(e) = self.role.initiate_noop_commit(&self.ctx, &self.role_tx).await { warn!(?e, "initiate_noop_commit failed — stepping down"); self.role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } @@ -580,9 +579,8 @@ where } Ok(other) => { self.role_tx.send(other).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to resend role event: {}", error_str); - crate::Error::Fatal(error_str) + error!("Failed to resend role event: {:?}", e); + crate::Error::Fatal(e.to_string()) })?; break; } @@ -597,28 +595,20 @@ where self.notify_new_commit(new_commit_data); } - RoleEvent::LeaderDiscovered(leader_id, term) => { debug!("LeaderDiscovered: leader_id={}, term={}", leader_id, term); // Notify leader change listeners - no state transition // Note: mpsc channels do not deduplicate; consumers handle dedup if needed self.notify_leader_change(Some(leader_id), term); } - RoleEvent::ReprocessEvent(raft_event) => { info!("Replay the RaftEvent: {:?}", &raft_event); - self.event_tx.send(*raft_event).await.map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) - })?; + self.buffered_raft_event.push_front(*raft_event); } - RoleEvent::LogFlushed { durable_index } => { debug!("LogFlushed: durable_index={}", durable_index); self.role.handle_log_flushed(durable_index, &self.ctx, &self.role_tx).await; } - RoleEvent::AppendResult { follower_id, result, @@ -632,19 +622,16 @@ where error!("handle_append_result failed: {:?}", e); } } - RoleEvent::NoopCommitted { term } => { debug!("NoopCommitted: term={}", term); // on_noop_committed already called directly in drain_commit_actions. // Only notify leader change listeners here (requires Raft access). self.notify_leader_change(Some(self.node_id), term); } - RoleEvent::FatalError { source, error } => { error!(%self.node_id, %source, %error, "Fatal error from SM worker — shutting down"); return Err(crate::Error::Fatal(format!("{source}: {error}"))); } - RoleEvent::ApplyCompleted { last_index, results, @@ -662,12 +649,10 @@ where warn!(%self.node_id, ?e, "Non-fatal error in ApplyCompleted handler"); } } - RoleEvent::PeerStreamError { peer_id } => { debug!(%peer_id, "PeerStreamError: bidi stream disconnected, resetting next_index"); self.role.handle_peer_stream_error(peer_id); } - RoleEvent::ZombieDetected(node_id) => { debug!(%node_id, "ZombieDetected: forwarding to leader for BatchRemove"); if let Err(e) = @@ -676,7 +661,6 @@ where error!(%node_id, ?e, "handle_zombie_detected failed"); } } - RoleEvent::SnapshotPushCompleted { peer_id, success } => { debug!(%peer_id, %success, "SnapshotPushCompleted"); if success { @@ -694,6 +678,59 @@ where let policy = &self.ctx.node_config.retry.install_snapshot; self.role.handle_snapshot_push_completed(peer_id, success, policy, self.node_id); } + RoleEvent::CreateSnapshotEvent => { + if let Err(e) = self.role.handle_create_snapshot(&self.ctx, &self.role_tx).await { + if e.is_fatal() { + return Err(e); + } + error!(%self.node_id, ?e, "handle_create_snapshot failed"); + } + } + RoleEvent::SnapshotCreated(result) => { + if let Err(e) = + self.role.handle_snapshot_created(result, &self.ctx, &self.role_tx).await + { + if e.is_fatal() { + return Err(e); + } + error!(%self.node_id, ?e, "handle_snapshot_created failed"); + } + } + RoleEvent::StepDownSelfRemoved => { + if let Err(e) = self.role.handle_self_removed(&self.role_tx) { + if e.is_fatal() { + return Err(e); + } + error!(%self.node_id, ?e, "handle_self_removed failed"); + } + } + RoleEvent::MembershipApplied => { + if let Err(e) = self.role.handle_membership_applied(&self.ctx, &self.role_tx).await + { + if e.is_fatal() { + return Err(e); + } + error!(%self.node_id, ?e, "handle_membership_applied failed"); + } + } + RoleEvent::PromoteReadyLearners => { + if let Err(e) = + self.role.handle_promote_ready_learners(&self.ctx, &self.role_tx).await + { + if e.is_fatal() { + return Err(e); + } + error!(%self.node_id, ?e, "handle_promote_ready_learners failed"); + } + } + RoleEvent::LogPurgeCompleted(log_id) => { + if let Err(e) = self.role.handle_log_purge_completed(log_id) { + if e.is_fatal() { + return Err(e); + } + error!(%self.node_id, ?e, "handle_log_purge_completed failed"); + } + } }; Ok(()) diff --git a/d-engine-core/src/raft_role/candidate_state.rs b/d-engine-core/src/raft_role/candidate_state.rs index 5e44a209..8d587cc9 100644 --- a/d-engine-core/src/raft_role/candidate_state.rs +++ b/d-engine-core/src/raft_role/candidate_state.rs @@ -375,9 +375,8 @@ impl RaftRoleState for CandidateState { sender .send(Err(Status::permission_denied("Not Follower or Learner."))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Err(ConsensusError::RoleViolation { @@ -391,51 +390,14 @@ impl RaftRoleState for CandidateState { .into()); } - RaftEvent::CreateSnapshotEvent => { - return Err(ConsensusError::RoleViolation { - current_role: "Candidate", - required_role: "Leader", - context: format!( - "Candidate node {} attempted to create snapshot.", - ctx.node_id - ), - } - .into()); - } - - RaftEvent::SnapshotCreated(_result) => { - return Err(ConsensusError::RoleViolation { - current_role: "Candidate", - required_role: "Leader", - context: format!( - "Candidate node {} attempted to handle created snapshot.", - ctx.node_id - ), - } - .into()); - } - - RaftEvent::LogPurgeCompleted(_purged_id) => { - return Err(ConsensusError::RoleViolation { - current_role: "Learner", - required_role: "Leader", - context: format!( - "Learner node {} should not receive LogPurgeCompleted event.", - ctx.node_id - ), - } - .into()); - } - RaftEvent::JoinCluster(_join_request, sender) => { sender .send(Err(Status::permission_denied( "Candidate should not receive JoinCluster event.", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Err(ConsensusError::RoleViolation { @@ -456,9 +418,8 @@ impl RaftRoleState for CandidateState { "Candidate should not response DiscoverLeader event.", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Ok(()); @@ -477,35 +438,12 @@ impl RaftRoleState for CandidateState { return Ok(()); } - RaftEvent::PromoteReadyLearners => { - return Err(ConsensusError::RoleViolation { - current_role: "Candidate", - required_role: "Leader", - context: format!( - "Candidate node {} receives RaftEvent::PromoteReadyLearners", - ctx.node_id - ), - } - .into()); - } - - RaftEvent::MembershipApplied => { - // Candidates don't maintain cluster metadata cache - // This event is only relevant for leaders - trace!("Candidate ignoring MembershipApplied event"); - } - RaftEvent::FatalError { source, error } => { error!("[Candidate] Fatal error from {}: {}", source, error); return Err(crate::Error::Fatal(format!( "Fatal error from {source}: {error}" ))); } - - RaftEvent::StepDownSelfRemoved => { - // Unreachable: handled at Raft level before reaching RoleState - unreachable!("StepDownSelfRemoved should be handled in Raft::run()"); - } } Ok(()) @@ -558,9 +496,8 @@ impl CandidateState { role_tx: &mpsc::UnboundedSender, ) -> Result<()> { role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str).into() + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)).into() }) } @@ -571,9 +508,8 @@ impl CandidateState { ) -> Result<()> { debug!("send_replay_raft_event, raft_event:{:?}", &raft_event); role_tx.send(RoleEvent::ReprocessEvent(Box::new(raft_event))).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str).into() + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)).into() }) } diff --git a/d-engine-core/src/raft_role/candidate_state_test.rs b/d-engine-core/src/raft_role/candidate_state_test.rs index 45b3de91..3844f31a 100644 --- a/d-engine-core/src/raft_role/candidate_state_test.rs +++ b/d-engine-core/src/raft_role/candidate_state_test.rs @@ -137,9 +137,9 @@ async fn test_candidate_drain_read_buffer_returns_error() { ); if let Err(e) = result { - let error_str = format!("{e:?}"); + let error_str = e.to_string(); assert!( - error_str.contains("NotLeader"), + error_str.contains("Membership changes require leader role"), "Error should be NotLeader, got: {error_str}" ); } @@ -957,40 +957,40 @@ async fn test_handle_discover_leader_returns_permission_denied() { } #[cfg(test)] -mod role_violation_tests { +mod snapshot_tests { use super::*; - /// Test: Role violation events return RoleViolation error + /// Test: Candidate rejects snapshot lifecycle events with RoleViolation /// - /// Original: test_role_violation_events + /// Candidate is a transient election state — snapshot operations are only + /// valid on stable roles (Follower, Learner, Leader). All three methods + /// use the trait default which returns `ConsensusError::RoleViolation`. #[tokio::test] - async fn test_candidate_role_violation_errors() { + async fn test_candidate_rejects_snapshot_events() { let (_graceful_tx, graceful_rx) = watch::channel(()); let context = mock_raft_context("/tmp/test_role_violation", graceful_rx, None); let mut state = CandidateState::::new(1, context.node_config.clone()); - - // Test CreateSnapshotEvent let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::CreateSnapshotEvent; - let e = state.handle_raft_event(raft_event, &context, role_tx).await.unwrap_err(); + + // CreateSnapshotEvent → RoleViolation + let e = state.handle_create_snapshot(&context, &role_tx).await.unwrap_err(); assert!(matches!( e, Error::Consensus(ConsensusError::RoleViolation { .. }) )); - // Test SnapshotCreated - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::SnapshotCreated(Err(Error::Fatal("test".to_string()))); - let e = state.handle_raft_event(raft_event, &context, role_tx).await.unwrap_err(); + // SnapshotCreated → RoleViolation + let e = state + .handle_snapshot_created(Err(Error::Fatal("test".to_string())), &context, &role_tx) + .await + .unwrap_err(); assert!(matches!( e, Error::Consensus(ConsensusError::RoleViolation { .. }) )); - // Test LogPurgeCompleted - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::LogPurgeCompleted(LogId { term: 1, index: 1 }); - let e = state.handle_raft_event(raft_event, &context, role_tx).await.unwrap_err(); + // LogPurgeCompleted → RoleViolation + let e = state.handle_log_purge_completed(LogId { term: 1, index: 1 }).unwrap_err(); assert!(matches!( e, Error::Consensus(ConsensusError::RoleViolation { .. }) diff --git a/d-engine-core/src/raft_role/follower_state.rs b/d-engine-core/src/raft_role/follower_state.rs index 7a108568..4169bdd2 100644 --- a/d-engine-core/src/raft_role/follower_state.rs +++ b/d-engine-core/src/raft_role/follower_state.rs @@ -1,16 +1,16 @@ -use std::fmt::Debug; -use std::marker::PhantomData; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - +use async_trait::async_trait; use d_engine_proto::common::LogId; use d_engine_proto::common::NodeRole::Follower; use d_engine_proto::server::cluster::ClusterConfUpdateResponse; use d_engine_proto::server::election::VoteResponse; - -use async_trait::async_trait; use d_engine_proto::server::storage::SnapshotAck; +use d_engine_proto::server::storage::SnapshotMetadata; use d_engine_proto::server::storage::SnapshotResponse; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use tokio::sync::mpsc; use tokio::time::Instant; use tonic::Status; @@ -172,9 +172,8 @@ impl RaftRoleState for FollowerState { debug!("follower::start_election..."); role_tx.send(RoleEvent::BecomeCandidate).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; Ok(()) @@ -242,9 +241,8 @@ impl RaftRoleState for FollowerState { ); sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } Err(e) => { @@ -255,9 +253,8 @@ impl RaftRoleState for FollowerState { last_log_term, }; sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; error("handle_raft_event::RaftEvent::ReceiveVoteRequest", &e); return Err(e); @@ -273,9 +270,8 @@ impl RaftRoleState for FollowerState { debug!("Follower receive ClusterConf: {:?}", &cluster_conf); sender.send(Ok(cluster_conf)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } @@ -316,9 +312,8 @@ impl RaftRoleState for FollowerState { my_id, &response ); sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } RaftEvent::AppendEntries(append_entries_request, sender) => { @@ -393,85 +388,14 @@ impl RaftRoleState for FollowerState { } } } - - RaftEvent::LogPurgeCompleted(_purged_id) => { - return Err(ConsensusError::RoleViolation { - current_role: "Follower", - required_role: "Leader", - context: format!( - "Follower node {} should not receive LogPurgeCompleted event.", - ctx.node_id - ), - } - .into()); - } - - RaftEvent::CreateSnapshotEvent => { - // Prevent duplicate snapshot creation - if self.snapshot_in_progress.load(Ordering::Acquire) { - info!( - "Follower snapshot creation already in progress. Skipping duplicate request." - ); - return Ok(()); - } - - self.snapshot_in_progress.store(true, Ordering::Release); - let state_machine_handler = ctx.state_machine_handler().clone(); - - // Use spawn to perform snapshot creation in the background - tokio::spawn(async move { - let result = state_machine_handler.create_snapshot().await; - info!( - "Follower SnapshotCreated event will be processed in another event thread" - ); - if let Err(e) = super::role_state::send_replay_raft_event( - &role_tx, - RaftEvent::SnapshotCreated(result), - ) { - error!("Follower failed to send snapshot creation result: {}", e); - } - }); - } - - RaftEvent::SnapshotCreated(result) => { - // Reset snapshot_in_progress flag - self.snapshot_in_progress.store(false, Ordering::SeqCst); - - // Per Raft §7: Follower independently purges logs after snapshot generation - match result { - Ok((metadata, _path)) => { - if let Some(last_included) = metadata.last_included { - info!(?last_included, "Follower snapshot created, purging logs"); - - // Follower independently decides to purge after snapshot - if self.can_purge_logs(self.last_purged_index, last_included) { - match ctx.purge_executor().execute_purge(last_included).await { - Ok(_) => { - self.last_purged_index = Some(last_included); - info!(?last_included, "Follower logs purged successfully"); - } - Err(e) => { - error!(?e, "Failed to purge logs after snapshot"); - } - } - } - } - } - Err(e) => { - error!(?e, "Follower snapshot creation failed"); - } - } - } - RaftEvent::JoinCluster(_join_request, sender) => { sender .send(Err(Status::permission_denied( "Follower should not receive JoinCluster event.", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Err(ConsensusError::RoleViolation { @@ -492,9 +416,8 @@ impl RaftRoleState for FollowerState { "Follower should not response DiscoverLeader event.", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Ok(()); @@ -513,35 +436,12 @@ impl RaftRoleState for FollowerState { return Ok(()); } - RaftEvent::PromoteReadyLearners => { - return Err(ConsensusError::RoleViolation { - current_role: "Follower", - required_role: "Leader", - context: format!( - "Follower node {} receives RaftEvent::PromoteReadyLearners", - ctx.node_id - ), - } - .into()); - } - - RaftEvent::MembershipApplied => { - // Followers don't maintain cluster metadata cache - // This event is only relevant for leaders - trace!("Follower ignoring MembershipApplied event"); - } - RaftEvent::FatalError { source, error } => { error!("[Follower] Fatal error from {}: {}", source, error); return Err(crate::Error::Fatal(format!( "Fatal error from {source}: {error}" ))); } - - RaftEvent::StepDownSelfRemoved => { - // Unreachable: handled at Raft level before reaching RoleState - unreachable!("StepDownSelfRemoved should be handled in Raft::run()"); - } } Ok(()) @@ -563,6 +463,104 @@ impl RaftRoleState for FollowerState { role_tx, ) } + + /// Trigger an independent snapshot on this role (Raft §7 — each server snapshots + /// independently). Called when `should_snapshot()` returns true after SM apply. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_create_snapshot( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Prevent duplicate snapshot creation + if self.snapshot_in_progress.load(Ordering::Acquire) { + info!("Snapshot creation already in progress. Skipping duplicate request."); + return Ok(()); + } + + self.snapshot_in_progress.store(true, Ordering::Release); + let state_machine_handler = ctx.state_machine_handler().clone(); + + // Use spawn to perform snapshot creation in the background + let role_tx = role_tx.clone(); + tokio::spawn(async move { + let result = state_machine_handler.create_snapshot().await; + info!("SnapshotCreated event will be processed in another event thread"); + if let Err(e) = role_tx.send(RoleEvent::SnapshotCreated(result)) { + error!("Follower failed to send snapshot creation result: {e:?}"); + } + }); + + Ok(()) + } + + /// Process the completed snapshot result (success or error). + /// Leader: schedules log purge up to `last_included`. + /// Follower/Learner: updates local snapshot path. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_snapshot_created( + &mut self, + result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, + ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Reset snapshot_in_progress flag + self.snapshot_in_progress.store(false, Ordering::SeqCst); + + // Per Raft §7: Follower independently purges logs after snapshot generation + match result { + Ok((metadata, _path)) => { + if let Some(last_included) = metadata.last_included { + info!(?last_included, "Follower snapshot created, purging logs"); + + // Follower independently decides to purge after snapshot + if self.can_purge_logs(self.last_purged_index, last_included) { + match ctx.purge_executor().execute_purge(last_included).await { + Ok(_) => { + self.last_purged_index = Some(last_included); + info!(?last_included, "Follower logs purged successfully"); + } + Err(e) => { + error!(?e, "Failed to purge logs after snapshot"); + } + } + } + } + } + Err(e) => { + error!(?e, "Follower snapshot creation failed"); + } + } + + Ok(()) + } + + // Stale in-flight events — these are Leader-only operations that may arrive + // after a step-down due to role_tx (unbounded, P2) race with BecomeFollower. + // Both orderings are valid; Follower silently ignores them rather than erroring. + + fn handle_log_purge_completed( + &mut self, + _purged_id: d_engine_proto::common::LogId, + ) -> Result<()> { + Ok(()) + } + + async fn handle_promote_ready_learners( + &mut self, + _ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + Ok(()) + } + + async fn handle_membership_applied( + &mut self, + _ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + Ok(()) + } } impl FollowerState { diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index 8b851c67..f3cd572e 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -1556,200 +1556,219 @@ fn test_can_purge_logs_handles_initial_state() { } // ============================================================================ -// Role Violation Tests Module +// Snapshot Tests Module // ============================================================================ -mod role_violation_tests { +mod snapshot_tests { use super::*; - use crate::ConsensusError; + use std::sync::atomic::Ordering; - /// Test: FollowerState rejects role violation events + /// Test: Follower gracefully ignores stale leader-only internal events /// - /// Scenario: - /// - Follower receives events that only Leader can handle: - /// - CreateSnapshotEvent - /// - SnapshotCreated - /// - LogPurgeCompleted - /// - /// Expected: - /// - All return Error::Consensus(ConsensusError::RoleViolation) - /// - handle_raft_event returns Err() - /// - No state changes + /// Protocol scenario: a leader steps down to follower while internal events + /// (LogPurgeCompleted, PromoteReadyLearners, StepDownSelfRemoved, MembershipApplied) + /// are still queued in role_rx. All must be silently ignored — returning an error + /// here would trigger a non-fatal warning log and waste a loop iteration for no reason. /// - /// This validates role-based access control for Raft events. - /// - /// Original: test_role_violation_events (in module) + /// Expected: all return Ok(()) — no panic, no state change. #[tokio::test] - async fn test_follower_rejects_leader_only_events() { + async fn test_follower_ignores_stale_leader_internal_events() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - - // [Test LogPurgeCompleted] - // Follower now handles CreateSnapshotEvent and SnapshotCreated independently (Raft §7) - // but should NOT receive LogPurgeCompleted from external sources let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::LogPurgeCompleted(LogId { term: 1, index: 1 }); - let e = state.handle_raft_event(raft_event, &context, role_tx).await.unwrap_err(); + // LogPurgeCompleted: leader-only, but stale events after step-down must not error assert!( - matches!(e, Error::Consensus(ConsensusError::RoleViolation { .. })), - "LogPurgeCompleted should return RoleViolation" + state.handle_log_purge_completed(LogId { term: 1, index: 1 }).is_ok(), + "Stale LogPurgeCompleted should be silently ignored" + ); + + // PromoteReadyLearners: leader-only, same reasoning + assert!( + state.handle_promote_ready_learners(&context, &role_tx).await.is_ok(), + "Stale PromoteReadyLearners should be silently ignored" + ); + + // StepDownSelfRemoved: only leader can self-remove, stale event must not error + assert!( + state.handle_self_removed(&role_tx).is_ok(), + "Stale StepDownSelfRemoved should be silently ignored" + ); + + // MembershipApplied: follower has no cache to refresh — pure no-op + assert!( + state.handle_membership_applied(&context, &role_tx).await.is_ok(), + "MembershipApplied on follower should be a no-op" ); } - /// Test: Follower ignores duplicate CreateSnapshotEvent while snapshot is in progress - /// - /// Purpose: - /// Validates that the snapshot_in_progress flag prevents concurrent snapshot creation, - /// ensuring snapshot consistency and avoiding resource waste from duplicate operations. + /// Test: Follower ignores duplicate CreateSnapshot while one is already in progress /// - /// Scenario: - /// - First CreateSnapshotEvent is received and sets snapshot_in_progress=true - /// - Second CreateSnapshotEvent arrives before first completes + /// The `snapshot_in_progress` flag guards against concurrent snapshot creation. + /// A second trigger (e.g. from two rapid ApplyCompleted events) must be silently + /// dropped rather than spawning a second background task. /// /// Expected: - /// - First event: Returns Ok(), starts async snapshot creation - /// - Second event: Returns Ok() immediately without starting new snapshot (logged as skipped) - /// - snapshot_in_progress flag protects against concurrent creation + /// - First call: Ok(), sets snapshot_in_progress = true, spawns background task + /// - Second call: Ok(), skips (flag already set), flag remains true #[tokio::test] - async fn test_follower_ignores_duplicate_create_snapshot_event() { + async fn test_follower_ignores_duplicate_create_snapshot() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - - // First CreateSnapshotEvent - should succeed let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result1 = state - .handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx.clone()) - .await; - assert!(result1.is_ok(), "First CreateSnapshotEvent should succeed"); - // Verify flag is set + // First trigger — starts background snapshot + let result1 = state.handle_create_snapshot(&context, &role_tx).await; assert!( - state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), - "snapshot_in_progress should be true after first event" + result1.is_ok(), + "First handle_create_snapshot should succeed" + ); + assert!( + state.snapshot_in_progress.load(Ordering::SeqCst), + "snapshot_in_progress should be true after first trigger" ); - // Second CreateSnapshotEvent - should be ignored - let result2 = - state.handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx).await; + // Second trigger while first is still running — must be a no-op + let result2 = state.handle_create_snapshot(&context, &role_tx).await; assert!( result2.is_ok(), - "Second CreateSnapshotEvent should return Ok (ignored)" + "Second handle_create_snapshot should return Ok (skipped)" ); - - // Flag should still be true (first snapshot still in progress) assert!( - state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), + state.snapshot_in_progress.load(Ordering::SeqCst), "snapshot_in_progress should remain true" ); } - /// Test: Follower resets snapshot_in_progress flag after SnapshotCreated (success case) + /// Test: Follower resets snapshot_in_progress and updates last_purged_index on success /// - /// Purpose: - /// Validates that the snapshot_in_progress flag is correctly reset after snapshot completion, - /// allowing subsequent snapshots to be created when needed. + /// Per Raft §7, followers independently purge logs after a successful snapshot. + /// This test verifies both the flag lifecycle and the purge side-effect. /// /// Scenario: - /// - snapshot_in_progress is manually set to true (simulating ongoing snapshot) - /// - SnapshotCreated event with successful result is received + /// - snapshot_in_progress is pre-set to true (simulating an in-flight snapshot) + /// - SnapshotCreated arrives with a successful result (last_included = index 50) /// /// Expected: - /// - Event handler returns Ok() - /// - snapshot_in_progress flag is reset to false - /// - System is ready to accept new CreateSnapshotEvent - /// - /// This ensures the flag lifecycle is: false → true (on create) → false (on complete) + /// - snapshot_in_progress reset to false + /// - last_purged_index updated to Some(LogId { term: 1, index: 50 }) #[tokio::test] - async fn test_follower_resets_snapshot_flag_on_success() { + async fn test_follower_snapshot_created_success_resets_flag_and_purges_logs() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - // Simulate snapshot in progress - state.snapshot_in_progress.store(true, std::sync::atomic::Ordering::SeqCst); + state.snapshot_in_progress.store(true, Ordering::SeqCst); + // can_purge_logs requires last_included.index < commit_index + state.update_commit_index(100).unwrap(); - // Create successful snapshot result + let last_included = LogId { term: 1, index: 50 }; let metadata = d_engine_proto::server::storage::SnapshotMetadata { - last_included: Some(LogId { term: 1, index: 50 }), + last_included: Some(last_included), checksum: bytes::Bytes::new(), }; - let snapshot_result = Ok((metadata, std::path::PathBuf::from("/tmp/test_snapshot.bin"))); + let snapshot_result = Ok((metadata, std::path::PathBuf::from("/tmp/snap.bin"))); let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state - .handle_raft_event( - RaftEvent::SnapshotCreated(snapshot_result), - &context, - role_tx, - ) - .await; - - assert!(result.is_ok(), "SnapshotCreated should succeed"); + let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; - // Verify flag is reset + assert!(result.is_ok(), "handle_snapshot_created should succeed"); assert!( - !state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), - "snapshot_in_progress should be false after SnapshotCreated" + !state.snapshot_in_progress.load(Ordering::SeqCst), + "snapshot_in_progress must be false after completion" + ); + assert_eq!( + state.last_purged_index, + Some(last_included), + "last_purged_index must advance to snapshot's last_included after log purge" ); } - /// Test: Follower resets snapshot_in_progress flag after SnapshotCreated (failure case) + /// Test: Follower resets snapshot_in_progress on failure but does NOT purge logs /// - /// Purpose: - /// Validates that the snapshot_in_progress flag is reset even when snapshot creation fails, - /// allowing the system to retry snapshot creation later without being permanently blocked. - /// - /// Scenario: - /// - snapshot_in_progress is set to true - /// - SnapshotCreated event with error result is received + /// A failed snapshot must not advance the purge boundary — the logs are still needed. + /// The flag must still be cleared so the next ApplyCompleted can retry. /// /// Expected: - /// - Event handler returns Ok() (error is logged but not propagated) - /// - snapshot_in_progress flag is reset to false - /// - System can retry snapshot creation on next ApplyCompleted trigger - /// - /// This ensures failure recovery: the flag doesn't stay locked after an error + /// - snapshot_in_progress reset to false + /// - last_purged_index remains None (no purge on failure) + /// - handler returns Ok() (error is logged, not propagated) #[tokio::test] - async fn test_follower_resets_snapshot_flag_on_failure() { + async fn test_follower_snapshot_created_failure_resets_flag_no_purge() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); + state.snapshot_in_progress.store(true, Ordering::SeqCst); - // Simulate snapshot in progress - state.snapshot_in_progress.store(true, std::sync::atomic::Ordering::SeqCst); - - // Create failed snapshot result let snapshot_result = Err(Error::Fatal("Snapshot creation failed".to_string())); let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state - .handle_raft_event( - RaftEvent::SnapshotCreated(snapshot_result), - &context, - role_tx, - ) - .await; + let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; assert!( result.is_ok(), - "SnapshotCreated with error should return Ok" + "handle_snapshot_created should return Ok even on failure" + ); + assert!( + !state.snapshot_in_progress.load(Ordering::SeqCst), + "snapshot_in_progress must be cleared even after failure" + ); + assert_eq!( + state.last_purged_index, None, + "last_purged_index must not advance when snapshot failed" + ); + } + + /// Test: Complete snapshot lifecycle — create, complete, create again + /// + /// Validates the full flag cycle: false → true (create) → false (created ok) → true (create again). + /// Ensures the follower can take multiple snapshots over its lifetime without getting stuck. + #[tokio::test] + async fn test_follower_snapshot_lifecycle() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + let (role_tx, _role_rx) = mpsc::unbounded_channel(); + + // Phase 1: trigger snapshot + assert!(state.handle_create_snapshot(&context, &role_tx).await.is_ok()); + assert!( + state.snapshot_in_progress.load(Ordering::SeqCst), + "flag set after create" + ); + + // Phase 2: snapshot completes successfully + let metadata = d_engine_proto::server::storage::SnapshotMetadata { + last_included: Some(LogId { + term: 1, + index: 100, + }), + checksum: bytes::Bytes::new(), + }; + let ok_result = Ok((metadata, std::path::PathBuf::from("/tmp/snap1.bin"))); + assert!(state.handle_snapshot_created(ok_result, &context, &role_tx).await.is_ok()); + assert!( + !state.snapshot_in_progress.load(Ordering::SeqCst), + "flag cleared after created" ); - // Verify flag is reset even on failure + // Phase 3: second snapshot can now be triggered + assert!(state.handle_create_snapshot(&context, &role_tx).await.is_ok()); assert!( - !state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), - "snapshot_in_progress should be false after failed SnapshotCreated" + state.snapshot_in_progress.load(Ordering::SeqCst), + "flag set again for second snapshot" ); } } @@ -2091,8 +2110,8 @@ async fn test_follower_handles_fatal_error_returns_error() { /// - State machine handler indicates snapshot should be taken /// /// Expected: -/// - CreateSnapshotEvent is sent back to role event loop for processing -/// - Event is reprocessed as RoleEvent::ReprocessEvent +/// - RoleEvent::CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) +/// - No ReprocessEvent wrapper — direct send eliminates the bounded event_tx deadlock path #[tokio::test] async fn test_apply_completed_triggers_snapshot_when_condition_met() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -2129,19 +2148,14 @@ async fn test_apply_completed_triggers_snapshot_when_condition_met() { "ApplyCompleted should be handled successfully, got: {result:?}" ); - // VERIFY 2: CreateSnapshotEvent is sent as RoleEvent::ReprocessEvent - let event = role_rx.try_recv().expect("Should receive snapshot event"); - match event { - RoleEvent::ReprocessEvent(boxed_event) => { - match *boxed_event { - RaftEvent::CreateSnapshotEvent => { - // Success! Event is correctly wrapped - } - other => panic!("Expected CreateSnapshotEvent, got: {other:?}"), - } - } - other => panic!("Expected RoleEvent::ReprocessEvent, got: {other:?}"), - } + // VERIFY 2: CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) + // check_and_trigger_snapshot no longer wraps in ReprocessEvent — direct send avoids + // the extra round-trip through event_tx that caused the original deadlock risk. + let event = role_rx.try_recv().expect("Should receive snapshot trigger event"); + assert!( + matches!(event, RoleEvent::CreateSnapshotEvent), + "Expected RoleEvent::CreateSnapshotEvent, got: {event:?}" + ); // VERIFY 3: No additional events queued assert!( diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 7de086da..260ea194 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -1146,9 +1146,8 @@ impl RaftRoleState for LeaderState { last_log_term: last_log_id.term, }; sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } } @@ -1186,9 +1185,8 @@ impl RaftRoleState for LeaderState { ); sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } else { // Step down as Follower as new Leader found @@ -1252,9 +1250,8 @@ impl RaftRoleState for LeaderState { sender .send(Err(Status::permission_denied("Not Follower or Learner. "))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Err(ConsensusError::RoleViolation { @@ -1268,98 +1265,6 @@ impl RaftRoleState for LeaderState { .into()); } - RaftEvent::CreateSnapshotEvent => { - // Prevent duplicate snapshot creation - if self.snapshot_in_progress.load(std::sync::atomic::Ordering::Acquire) { - info!("Snapshot creation already in progress. Skipping duplicate request."); - return Ok(()); - } - - self.snapshot_in_progress.store(true, std::sync::atomic::Ordering::Release); - let state_machine_handler = ctx.state_machine_handler().clone(); - - // Use spawn to perform snapshot creation in the background - tokio::spawn(async move { - let result = state_machine_handler.create_snapshot().await; - info!("SnapshotCreated event will be processed in another event thread"); - if let Err(e) = - send_replay_raft_event(&role_tx, RaftEvent::SnapshotCreated(result)) - { - error!("Failed to send snapshot creation result: {}", e); - } - }); - } - - RaftEvent::SnapshotCreated(result) => { - self.snapshot_in_progress.store(false, Ordering::SeqCst); - - match result { - Err(e) => { - error!(%e, "State machine snapshot creation failed"); - } - Ok(( - SnapshotMetadata { - last_included: last_included_option, - checksum: _, - }, - _final_path, - )) => { - info!("Initiating log purge after snapshot creation"); - - if let Some(last_included) = last_included_option { - // ---------------------- - // Phase 1: Schedule log purge if possible - // ---------------------- - trace!("Phase 1: Schedule log purge if possible"); - if self.can_purge_logs(self.last_purged_index, last_included) { - trace!(?last_included, "Phase 1: Scheduling log purge"); - self.scheduled_purge_upto(last_included); - } - - // ---------------------- - // Phase 2: Execute local purge - // ---------------------- - // Per Raft §7: Leader purges independently without peer coordination - trace!("Phase 2: Execute scheduled purge task"); - debug!(?last_included, "Execute scheduled purge task"); - if let Some(scheduled) = self.scheduled_purge_upto { - let purge_executor = ctx.purge_executor(); - match purge_executor.execute_purge(scheduled).await { - Ok(_) => { - if let Err(e) = send_replay_raft_event( - &role_tx, - RaftEvent::LogPurgeCompleted(scheduled), - ) { - error!(%e, "Failed to notify purge completion"); - } - } - Err(e) => { - error!(?e, ?scheduled, "Log purge execution failed"); - } - } - } - } - } - } - } - - RaftEvent::LogPurgeCompleted(purged_id) => { - // Ensure we don't regress the purge index - if self.last_purged_index.map_or(true, |current| purged_id.index > current.index) { - debug!( - ?purged_id, - "Updating last purged index after successful execution" - ); - self.last_purged_index = Some(purged_id); - } else { - warn!( - ?purged_id, - ?self.last_purged_index, - "Received outdated purge completion, ignoring" - ); - } - } - RaftEvent::JoinCluster(join_request, sender) => { debug!(?join_request, "Leader::RaftEvent::JoinCluster"); self.handle_join_cluster(join_request, sender, ctx, &role_tx).await?; @@ -1375,9 +1280,8 @@ impl RaftRoleState for LeaderState { term: my_term, }; sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Ok(()); } else { @@ -1428,80 +1332,6 @@ impl RaftRoleState for LeaderState { // chunk_tx dropped } } - RaftEvent::PromoteReadyLearners => { - // SAFETY: Called from main event loop, no reentrancy issues - info!( - "[Leader {}] ⚡ PromoteReadyLearners event received, pending_promotions: {:?}", - self.node_id(), - self.pending_promotions.iter().map(|p| p.node_id).collect::>() - ); - self.process_pending_promotions(ctx, &role_tx).await?; - } - - RaftEvent::MembershipApplied => { - // Save old membership for comparison - let old_replication_targets = self.cluster_metadata.replication_targets.clone(); - - // Refresh cluster metadata cache after membership change is applied - debug!("Refreshing cluster metadata cache after membership change"); - if let Err(e) = self.update_cluster_metadata(&ctx.membership()).await { - warn!("Failed to update cluster metadata: {:?}", e); - } - - // CRITICAL FIX #218: Initialize replication state for newly added peers - // Per Raft protocol: When new members join, Leader must initialize their next_index - let newly_added: Vec = self - .cluster_metadata - .replication_targets - .iter() - .filter(|new_peer| { - !old_replication_targets.iter().any(|old_peer| old_peer.id == new_peer.id) - }) - .map(|peer| peer.id) - .collect(); - - if !newly_added.is_empty() { - debug!( - "Initializing replication state for {} new peer(s): {:?}", - newly_added.len(), - newly_added - ); - let last_entry_id = ctx.raft_log().last_entry_id(); - if let Err(e) = self - .init_peers_next_index_and_match_index(last_entry_id, newly_added.clone()) - { - warn!("Failed to initialize next_index for new peers: {:?}", e); - // Non-fatal: next_index will use default value of 1, - // replication will still work but may be less efficient - } else { - trace!( - "[MEMBERSHIP-APPLIED-LEADER] Successfully initialized next_index/match_index for peers: {:?}", - newly_added - ); - } - } else { - trace!("[MEMBERSHIP-APPLIED-LEADER] No newly added peers detected"); - } - - // Drop workers for peers that were removed from membership. - // Dropping ReplicationWorkerHandle closes task_tx → worker exits naturally. - let removed: Vec = old_replication_targets - .iter() - .filter(|old_peer| { - !self - .cluster_metadata - .replication_targets - .iter() - .any(|new_peer| new_peer.id == old_peer.id) - }) - .map(|peer| peer.id) - .collect(); - for removed_id in &removed { - if self.replication_workers.remove(removed_id).is_some() { - debug!("Dropped replication worker for removed peer {}", removed_id); - } - } - } RaftEvent::FatalError { source, error } => { error!("[Leader] Fatal error from {}: {}", source, error); @@ -1565,26 +1395,6 @@ impl RaftRoleState for LeaderState { "Fatal error from {source}: {error}" ))); } - - RaftEvent::StepDownSelfRemoved => { - // Only Leader can propose configuration changes and remove itself - // Per Raft protocol: Leader steps down immediately after self-removal - warn!( - "[Leader-{}] Removed from cluster membership, stepping down to Follower", - self.node_id() - ); - role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { - error!( - "[Leader-{}] Failed to send BecomeFollower after self-removal: {:?}", - self.node_id(), - e - ); - NetworkError::SingalSendFailed(format!( - "BecomeFollower after self-removal: {e:?}" - )) - })?; - return Ok(()); - } } Ok(()) @@ -1923,6 +1733,345 @@ impl RaftRoleState for LeaderState { Ok(()) } + + /// Advance the purge boundary after log entries up to `purged_id` are removed. + /// Leader only: updates `last_purged_index`. + /// Default: no-op + warn (unexpected on non-leader). + fn handle_log_purge_completed( + &mut self, + purged_id: LogId, + ) -> Result<()> { + // Ensure we don't regress the purge index + if self.last_purged_index.is_none_or(|current| purged_id.index > current.index) { + debug!( + ?purged_id, + "Updating last purged index after successful execution" + ); + self.last_purged_index = Some(purged_id); + } else { + warn!( + ?purged_id, + ?self.last_purged_index, + "Received outdated purge completion, ignoring" + ); + } + + Ok(()) + } + + /// Trigger an independent snapshot on this role (Raft §7 — each server snapshots + /// independently). Called when `should_snapshot()` returns true after SM apply. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_create_snapshot( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Prevent duplicate snapshot creation + if self.snapshot_in_progress.load(std::sync::atomic::Ordering::Acquire) { + info!("Snapshot creation already in progress. Skipping duplicate request."); + return Ok(()); + } + + self.snapshot_in_progress.store(true, std::sync::atomic::Ordering::Release); + let state_machine_handler = ctx.state_machine_handler().clone(); + + // Use spawn to perform snapshot creation in the background + let role_tx = role_tx.clone(); + tokio::spawn(async move { + let result = state_machine_handler.create_snapshot().await; + info!("SnapshotCreated event will be processed in another event thread"); + if let Err(e) = role_tx.send(RoleEvent::SnapshotCreated(result)) { + error!("Failed to send snapshot creation result: {e:?}"); + } + }); + + Ok(()) + } + /// Process the completed snapshot result (success or error). + /// Leader: schedules log purge up to `last_included`. + /// Follower/Learner: updates local snapshot path. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_snapshot_created( + &mut self, + result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + self.snapshot_in_progress.store(false, Ordering::SeqCst); + + match result { + Err(e) => { + error!(%e, "State machine snapshot creation failed"); + } + Ok(( + SnapshotMetadata { + last_included: last_included_option, + checksum: _, + }, + _final_path, + )) => { + info!("Initiating log purge after snapshot creation"); + + if let Some(last_included) = last_included_option { + // ---------------------- + // Phase 1: Schedule log purge if possible + // ---------------------- + trace!("Phase 1: Schedule log purge if possible"); + if self.can_purge_logs(self.last_purged_index, last_included) { + trace!(?last_included, "Phase 1: Scheduling log purge"); + self.scheduled_purge_upto(last_included); + } + + // ---------------------- + // Phase 2: Execute local purge + // ---------------------- + // Per Raft §7: Leader purges independently without peer coordination + trace!("Phase 2: Execute scheduled purge task"); + debug!(?last_included, "Execute scheduled purge task"); + if let Some(scheduled) = self.scheduled_purge_upto { + let purge_executor = ctx.purge_executor(); + match purge_executor.execute_purge(scheduled).await { + Ok(_) => { + if let Err(e) = + role_tx.send(RoleEvent::LogPurgeCompleted(scheduled)) + { + error!(%e, "Failed to notify purge completion"); + } + } + Err(e) => { + error!(?e, ?scheduled, "Log purge execution failed"); + } + } + } + } + } + } + Ok(()) + } + + /// Check pending learners for promotion eligibility after a membership change. + /// Leader only: evaluates `pending_promotions`, proposes config change if ready. + /// Default: no-op + warn (unexpected on non-leader). + async fn handle_promote_ready_learners( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // SAFETY: Called from main event loop, no reentrancy issues + info!( + "[Leader {}] PromoteReadyLearners event received, pending_promotions: {:?}", + self.node_id(), + self.pending_promotions.iter().map(|p| p.node_id).collect::>() + ); + + // Get promotion configuration from the node config + let config = &ctx.node_config().raft.membership.promotion; + + // Step 1: Remove stale entries (older than configured threshold) + let now = Instant::now(); + self.pending_promotions.retain(|entry| { + now.saturating_duration_since(entry.ready_since) <= config.stale_learner_threshold + }); + // Refresh deadline after potential removals + self.refresh_stale_deadline(config.stale_learner_threshold); + + if self.pending_promotions.is_empty() { + debug!( + "[Leader {}] ❌ pending_promotions is empty after stale cleanup", + self.node_id() + ); + return Ok(()); + } + + // Step 2: Get current voter count (including self as leader) + let membership = ctx.membership(); + let current_voters = membership.voters().await.len() + 1; // +1 for self (leader) + debug!( + "[Leader {}] 📊 current_voters: {}, pending: {}", + self.node_id(), + current_voters, + self.pending_promotions.len() + ); + + // Step 3: Calculate the maximum batch size that preserves an odd total + let max_batch_size = + calculate_safe_batch_size(current_voters, self.pending_promotions.len()); + debug!( + "[Leader {}] 🎯 max_batch_size: {}", + self.node_id(), + max_batch_size + ); + + if max_batch_size == 0 { + // Nothing we can safely promote now + debug!( + "[Leader {}] max_batch_size is 0, cannot promote now", + self.node_id() + ); + return Ok(()); + } + + // Step 4: Extract the batch from the queue (FIFO order) + let promotion_entries = self.drain_batch(max_batch_size); + let promotion_node_ids = promotion_entries.iter().map(|e| e.node_id).collect::>(); + + // Step 5: Execute batch promotion + if !promotion_node_ids.is_empty() { + // Log the batch promotion + info!( + "Promoting learner batch of {} nodes: {:?} (total voters: {} -> {})", + promotion_node_ids.len(), + promotion_node_ids, + current_voters, + current_voters + promotion_node_ids.len() + ); + + // Attempt promotion and restore batch on failure + let result = self.safe_batch_promote(promotion_node_ids.clone(), ctx, role_tx).await; + + if let Err(e) = result { + // Restore entries to the front of the queue in reverse order + for entry in promotion_entries.into_iter().rev() { + self.pending_promotions.push_front(entry); + } + // Refresh deadline to account for restored entries at front + let threshold = config.stale_learner_threshold; + self.refresh_stale_deadline(threshold); + return Err(e); + } + + // Refresh stale deadline to reflect the new queue front after successful drain + let threshold = config.stale_learner_threshold; + self.refresh_stale_deadline(threshold); + + info!( + "Promotion successful. Cluster members: {:?}", + membership.voters().await + ); + } + + trace!( + ?self.pending_promotions, + "Step 6: Reschedule if any pending promotions remain" + ); + // Step 6: Reschedule if any pending promotions remain + if !self.pending_promotions.is_empty() { + debug!( + "[Leader {}] Re-sending PromoteReadyLearners for remaining pending: {:?}", + self.node_id(), + self.pending_promotions.iter().map(|p| p.node_id).collect::>() + ); + // Important: Re-send the event to trigger next cycle + role_tx.send(RoleEvent::PromoteReadyLearners).map_err(|e| { + error!("Send PromoteReadyLearners event failed: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) + })?; + } + + Ok(()) + } + + /// Membership config change applied to state — refresh any role-local derived state. + /// Leader: invalidates `cluster_metadata` cache. + /// Learner: checks if promoted to Voter; emits `BecomeFollower` if so. + /// Default: no-op (Follower/Candidate have no derived state to refresh). + async fn handle_membership_applied( + &mut self, + ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Save old membership for comparison + let old_replication_targets = self.cluster_metadata.replication_targets.clone(); + + // Refresh cluster metadata cache after membership change is applied + debug!("Refreshing cluster metadata cache after membership change"); + if let Err(e) = self.update_cluster_metadata(&ctx.membership()).await { + warn!("Failed to update cluster metadata: {:?}", e); + } + + // CRITICAL FIX #218: Initialize replication state for newly added peers + // Per Raft protocol: When new members join, Leader must initialize their next_index + let newly_added: Vec = self + .cluster_metadata + .replication_targets + .iter() + .filter(|new_peer| { + !old_replication_targets.iter().any(|old_peer| old_peer.id == new_peer.id) + }) + .map(|peer| peer.id) + .collect(); + + if !newly_added.is_empty() { + debug!( + "Initializing replication state for {} new peer(s): {:?}", + newly_added.len(), + newly_added + ); + let last_entry_id = ctx.raft_log().last_entry_id(); + if let Err(e) = + self.init_peers_next_index_and_match_index(last_entry_id, newly_added.clone()) + { + warn!("Failed to initialize next_index for new peers: {:?}", e); + // Non-fatal: next_index will use default value of 1, + // replication will still work but may be less efficient + } else { + trace!( + "[MEMBERSHIP-APPLIED-LEADER] Successfully initialized next_index/match_index for peers: {:?}", + newly_added + ); + } + } else { + trace!("[MEMBERSHIP-APPLIED-LEADER] No newly added peers detected"); + } + + // Drop workers for peers that were removed from membership. + // Dropping ReplicationWorkerHandle closes task_tx → worker exits naturally. + let removed: Vec = old_replication_targets + .iter() + .filter(|old_peer| { + !self + .cluster_metadata + .replication_targets + .iter() + .any(|new_peer| new_peer.id == old_peer.id) + }) + .map(|peer| peer.id) + .collect(); + + for removed_id in &removed { + if self.replication_workers.remove(removed_id).is_some() { + debug!("Dropped replication worker for removed peer {}", removed_id); + } + } + + Ok(()) + } + + /// Node was removed from cluster membership; step down immediately per Raft protocol. + /// Leader: emits `BecomeFollower`. Non-leader: unreachable in practice. + /// Default: warn + Ok(()) (defensive; only Leader should receive this). + fn handle_self_removed( + &mut self, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Only Leader can propose configuration changes and remove itself + // Per Raft protocol: Leader steps down immediately after self-removal + warn!( + "[Leader-{}] Removed from cluster membership, stepping down to Follower", + self.node_id() + ); + role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { + error!( + "[Leader-{}] Failed to send BecomeFollower after self-removal: {:?}", + self.node_id(), + e + ); + NetworkError::SingalSendFailed(format!("BecomeFollower after self-removal: {e:?}")) + })?; + + Ok(()) + } } /// Computes the exponential backoff delay for a snapshot push failure. @@ -2800,17 +2949,12 @@ impl LeaderState { self.refresh_stale_deadline(threshold); } - role_tx - .send(RoleEvent::ReprocessEvent(Box::new( - RaftEvent::PromoteReadyLearners, + role_tx.send(RoleEvent::PromoteReadyLearners).map_err(|e| { + error!("Failed to send PromoteReadyLearners: {e:?}"); + Error::System(SystemError::Network(NetworkError::SingalSendFailed( + e.to_string(), ))) - .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send PromoteReadyLearners: {}", error_str); - Error::System(SystemError::Network(NetworkError::SingalSendFailed( - error_str, - ))) - })?; + })?; Ok(()) } @@ -2915,9 +3059,8 @@ impl LeaderState { "Leader is going to step down as Follower..." ); role_tx.send(RoleEvent::BecomeFollower(new_leader_id)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; Ok(()) @@ -3228,130 +3371,6 @@ impl LeaderState { Ok(()) } - /// Processes all pending promotions while respecting the cluster's odd-node constraint - pub async fn process_pending_promotions( - &mut self, - ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, - ) -> Result<()> { - debug!( - "[Leader {}] 🔄 process_pending_promotions called, pending: {:?}", - self.node_id(), - self.pending_promotions.iter().map(|p| p.node_id).collect::>() - ); - - // Get promotion configuration from the node config - let config = &ctx.node_config().raft.membership.promotion; - - // Step 1: Remove stale entries (older than configured threshold) - let now = Instant::now(); - self.pending_promotions.retain(|entry| { - now.saturating_duration_since(entry.ready_since) <= config.stale_learner_threshold - }); - // Refresh deadline after potential removals - self.refresh_stale_deadline(config.stale_learner_threshold); - - if self.pending_promotions.is_empty() { - debug!( - "[Leader {}] ❌ pending_promotions is empty after stale cleanup", - self.node_id() - ); - return Ok(()); - } - - // Step 2: Get current voter count (including self as leader) - let membership = ctx.membership(); - let current_voters = membership.voters().await.len() + 1; // +1 for self (leader) - debug!( - "[Leader {}] 📊 current_voters: {}, pending: {}", - self.node_id(), - current_voters, - self.pending_promotions.len() - ); - - // Step 3: Calculate the maximum batch size that preserves an odd total - let max_batch_size = - calculate_safe_batch_size(current_voters, self.pending_promotions.len()); - debug!( - "[Leader {}] 🎯 max_batch_size: {}", - self.node_id(), - max_batch_size - ); - - if max_batch_size == 0 { - // Nothing we can safely promote now - debug!( - "[Leader {}] ⚠️ max_batch_size is 0, cannot promote now", - self.node_id() - ); - return Ok(()); - } - - // Step 4: Extract the batch from the queue (FIFO order) - let promotion_entries = self.drain_batch(max_batch_size); - let promotion_node_ids = promotion_entries.iter().map(|e| e.node_id).collect::>(); - - // Step 5: Execute batch promotion - if !promotion_node_ids.is_empty() { - // Log the batch promotion - info!( - "Promoting learner batch of {} nodes: {:?} (total voters: {} -> {})", - promotion_node_ids.len(), - promotion_node_ids, - current_voters, - current_voters + promotion_node_ids.len() - ); - - // Attempt promotion and restore batch on failure - let result = self.safe_batch_promote(promotion_node_ids.clone(), ctx, role_tx).await; - - if let Err(e) = result { - // Restore entries to the front of the queue in reverse order - for entry in promotion_entries.into_iter().rev() { - self.pending_promotions.push_front(entry); - } - // Refresh deadline to account for restored entries at front - let threshold = config.stale_learner_threshold; - self.refresh_stale_deadline(threshold); - return Err(e); - } - - // Refresh stale deadline to reflect the new queue front after successful drain - let threshold = config.stale_learner_threshold; - self.refresh_stale_deadline(threshold); - - info!( - "Promotion successful. Cluster members: {:?}", - membership.voters().await - ); - } - - trace!( - ?self.pending_promotions, - "Step 6: Reschedule if any pending promotions remain" - ); - // Step 6: Reschedule if any pending promotions remain - if !self.pending_promotions.is_empty() { - debug!( - "[Leader {}] Re-sending PromoteReadyLearners for remaining pending: {:?}", - self.node_id(), - self.pending_promotions.iter().map(|p| p.node_id).collect::>() - ); - // Important: Re-send the event to trigger next cycle - role_tx - .send(RoleEvent::ReprocessEvent(Box::new( - RaftEvent::PromoteReadyLearners, - ))) - .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Send PromoteReadyLearners event failed: {}", error_str); - NetworkError::SingalSendFailed(error_str) - })?; - } - - Ok(()) - } - /// Removes the first `count` nodes from the pending queue and returns them pub(super) fn drain_batch( &mut self, diff --git a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs index a6ad073a..8ae3e78d 100644 --- a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs @@ -958,9 +958,7 @@ async fn test_step_down_self_removed_sends_become_follower() { let mut leader_state = LeaderState::::new(1, node_config); let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let result = leader_state - .handle_raft_event(RaftEvent::StepDownSelfRemoved, &raft_context, role_tx) - .await; + let result = leader_state.handle_self_removed(&role_tx); assert!(result.is_ok(), "StepDownSelfRemoved must return Ok"); diff --git a/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs b/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs index 4b7b02da..d21962cf 100644 --- a/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs @@ -898,7 +898,7 @@ mod check_learner_progress_tests { use crate::MockMembership; use crate::RaftContext; - use crate::event::{RaftEvent, RoleEvent}; + use crate::event::RoleEvent; use crate::raft_role::leader_state::{LeaderState, PendingPromotion}; use crate::raft_role::role_state::RaftRoleState; use crate::test_utils::mock::MockTypeConfig; @@ -1176,9 +1176,7 @@ mod check_learner_progress_tests { // Should receive exactly 1 event let mut event_count = 0; while let Ok(event) = role_rx.try_recv() { - if let RoleEvent::ReprocessEvent(inner) = event - && matches!(*inner, RaftEvent::PromoteReadyLearners) - { + if matches!(event, RoleEvent::PromoteReadyLearners) { event_count += 1; } } @@ -1208,7 +1206,7 @@ mod pending_promotion_tests { use crate::MockRaftLog; use crate::MockReplicationCore; use crate::RaftContext; - use crate::event::{RaftEvent, RoleEvent}; + use crate::event::RoleEvent; use crate::raft_role::leader_state::{ LeaderState, PendingPromotion, calculate_safe_batch_size, }; @@ -1348,7 +1346,7 @@ mod pending_promotion_tests { // Result is Err(Timeout): quorum cannot complete without a running Raft loop let _ = fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await; // Verify the correct BatchPromote config was submitted with all 3 nodes @@ -1476,7 +1474,7 @@ mod pending_promotion_tests { let _ = fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await; // Verify FIFO: node 1 (older) was submitted, not node 2 @@ -1518,7 +1516,7 @@ mod pending_promotion_tests { let result = fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await; // Fire-and-forget: 9 submitted successfully, returns Ok @@ -1531,8 +1529,7 @@ mod pending_promotion_tests { let reschedule_count = { let mut count = 0; while let Ok(event) = fixture.role_rx.try_recv() { - if matches!(event, RoleEvent::ReprocessEvent(ref inner) if matches!(**inner, RaftEvent::PromoteReadyLearners)) - { + if matches!(event, RoleEvent::PromoteReadyLearners) { count += 1; } } @@ -1561,7 +1558,7 @@ mod pending_promotion_tests { (1..=2).map(|id| PendingPromotion::new(id, Instant::now())).collect(); let result = fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await; // Fire-and-forget: 1 submitted, returns Ok @@ -1585,7 +1582,7 @@ mod pending_promotion_tests { fixture.leader_state.update_current_term(2); let result = fixture .leader_state - .process_pending_promotions(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) .await; // Fire-and-forget: term change does not prevent submission diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs index 58031841..2c993ddf 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs @@ -52,14 +52,14 @@ async fn test_create_snapshot_event_starts_and_ignores_duplicates() { // First event should set the flag and spawn the task state - .handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx.clone()) + .handle_create_snapshot(&context, &role_tx) .await .expect("Should start snapshot creation"); assert!(state.snapshot_in_progress.load(Ordering::SeqCst)); // Second event should be ignored (flag still set) state - .handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx) + .handle_create_snapshot(&context, &role_tx) .await .expect("Should ignore duplicate snapshot creation"); // Still true, no panic, no duplicate task @@ -90,24 +90,33 @@ async fn test_snapshot_in_progress_flag_reset_on_created_event() { let mut state = LeaderState::::new(1, context.node_config()); state.snapshot_in_progress.store(true, Ordering::SeqCst); - // Success case - let raft_event = RaftEvent::SnapshotCreated(Ok(( - SnapshotMetadata { - last_included: Some(LogId { term: 1, index: 10 }), - checksum: "abc".into(), - }, - PathBuf::from("/tmp/fake"), - ))); let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let _ = state.handle_raft_event(raft_event, &context, role_tx).await; + + // Success case + let _ = state + .handle_snapshot_created( + Ok(( + SnapshotMetadata { + last_included: Some(LogId { term: 1, index: 10 }), + checksum: "abc".into(), + }, + PathBuf::from("/tmp/fake"), + )), + &context, + &role_tx, + ) + .await; assert!(!state.snapshot_in_progress.load(Ordering::SeqCst)); // Error case state.snapshot_in_progress.store(true, Ordering::SeqCst); - let raft_event = - RaftEvent::SnapshotCreated(Err(SnapshotError::OperationFailed("fail".into()).into())); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let _ = state.handle_raft_event(raft_event, &context, role_tx).await; + let _ = state + .handle_snapshot_created( + Err(SnapshotError::OperationFailed("fail".into()).into()), + &context, + &role_tx, + ) + .await; assert!(!state.snapshot_in_progress.load(Ordering::SeqCst)); } @@ -134,7 +143,7 @@ async fn test_create_snapshot_event_is_non_blocking() { let (role_tx, _role_rx) = mpsc::unbounded_channel(); let start = std::time::Instant::now(); - let _ = state.handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx).await; + let _ = state.handle_create_snapshot(&context, &role_tx).await; let elapsed = start.elapsed(); // Should return quickly (not wait for snapshot) @@ -162,15 +171,8 @@ async fn test_create_snapshot_event_is_non_blocking() { /// - Event handling returns error /// - No role transition events sent /// - snapshot_in_progress flag is reset -#[tokio::test] -async fn test_handle_log_purge_completed() { - let temp_dir = tempfile::tempdir().unwrap(); - let case_path = temp_dir.path().join("test_handle_log_purge_completed"); - - // Setup - let (_graceful_tx, graceful_rx) = watch::channel(()); - let context = MockBuilder::new(graceful_rx).with_db_path(&case_path).build_context(); - +#[test] +fn test_handle_log_purge_completed() { let mut state = LeaderState::::new(1, Arc::new(CoreRaftNodeConfig::default())); state.last_purged_index = Some(LogId { term: 1, @@ -178,13 +180,11 @@ async fn test_handle_log_purge_completed() { }); // Test updating to a higher index - let event = RaftEvent::LogPurgeCompleted(LogId { - term: 1, - index: 150, - }); state - .handle_raft_event(event, &context, mpsc::unbounded_channel().0) - .await + .handle_log_purge_completed(LogId { + term: 1, + index: 150, + }) .unwrap(); assert_eq!( state.last_purged_index, @@ -195,13 +195,11 @@ async fn test_handle_log_purge_completed() { ); // Test updating to a lower index (should be ignored) - let event = RaftEvent::LogPurgeCompleted(LogId { - term: 1, - index: 120, - }); state - .handle_raft_event(event, &context, mpsc::unbounded_channel().0) - .await + .handle_log_purge_completed(LogId { + term: 1, + index: 120, + }) .unwrap(); assert_eq!( state.last_purged_index, @@ -213,11 +211,7 @@ async fn test_handle_log_purge_completed() { // Test first purge state.last_purged_index = None; - let event = RaftEvent::LogPurgeCompleted(LogId { term: 1, index: 50 }); - state - .handle_raft_event(event, &context, mpsc::unbounded_channel().0) - .await - .unwrap(); + state.handle_log_purge_completed(LogId { term: 1, index: 50 }).unwrap(); assert_eq!(state.last_purged_index, Some(LogId { term: 1, index: 50 })); } diff --git a/d-engine-core/src/raft_role/learner_state.rs b/d-engine-core/src/raft_role/learner_state.rs index f5078258..04d7f07c 100644 --- a/d-engine-core/src/raft_role/learner_state.rs +++ b/d-engine-core/src/raft_role/learner_state.rs @@ -32,6 +32,7 @@ use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::storage::SnapshotAck; +use d_engine_proto::server::storage::SnapshotMetadata; use d_engine_proto::server::storage::SnapshotResponse; use std::fmt::Debug; use std::marker::PhantomData; @@ -210,9 +211,8 @@ impl RaftRoleState for LearnerState { ); sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } @@ -223,9 +223,8 @@ impl RaftRoleState for LearnerState { "Not able to respond to cluster conf request as node is Learner", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } @@ -266,9 +265,8 @@ impl RaftRoleState for LearnerState { my_id, &response ); sender.send(Ok(response)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } @@ -340,84 +338,14 @@ impl RaftRoleState for LearnerState { } } - RaftEvent::CreateSnapshotEvent => { - // Prevent duplicate snapshot creation - if self.snapshot_in_progress.load(Ordering::Acquire) { - info!( - "Learner snapshot creation already in progress. Skipping duplicate request." - ); - return Ok(()); - } - - self.snapshot_in_progress.store(true, Ordering::Release); - let state_machine_handler = ctx.state_machine_handler().clone(); - - // Use spawn to perform snapshot creation in the background - tokio::spawn(async move { - let result = state_machine_handler.create_snapshot().await; - info!( - "Learner SnapshotCreated event will be processed in another event thread" - ); - if let Err(e) = super::role_state::send_replay_raft_event( - &role_tx, - RaftEvent::SnapshotCreated(result), - ) { - error!("Learner failed to send snapshot creation result: {}", e); - } - }); - } - - RaftEvent::SnapshotCreated(result) => { - // Reset snapshot_in_progress flag - self.snapshot_in_progress.store(false, Ordering::SeqCst); - - // Per Raft §7: Learner independently purges logs after snapshot generation - match result { - Ok((metadata, _path)) => { - if let Some(last_included) = metadata.last_included { - info!(?last_included, "Learner snapshot created, purging logs"); - - // Learner independently decides to purge after snapshot - if self.can_purge_logs(self.last_purged_index, last_included) { - match ctx.purge_executor().execute_purge(last_included).await { - Ok(_) => { - self.last_purged_index = Some(last_included); - info!(?last_included, "Learner logs purged successfully"); - } - Err(e) => { - error!(?e, "Failed to purge logs after snapshot"); - } - } - } - } - } - Err(e) => { - error!(?e, "Learner snapshot creation failed"); - } - } - } - - RaftEvent::LogPurgeCompleted(_purged_id) => { - return Err(ConsensusError::RoleViolation { - current_role: "Learner", - required_role: "Leader", - context: format!( - "Learner node {} should not receive LogPurgeCompleted event.", - ctx.node_id - ), - } - .into()); - } - RaftEvent::JoinCluster(_join_request, sender) => { sender .send(Err(Status::permission_denied( "Learner should not receive JoinCluster event.", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Err(ConsensusError::RoleViolation { @@ -438,9 +366,8 @@ impl RaftRoleState for LearnerState { "Learner should not response DiscoverLeader event.", ))) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; return Ok(()); @@ -459,65 +386,12 @@ impl RaftRoleState for LearnerState { return Ok(()); } - RaftEvent::PromoteReadyLearners => { - return Err(ConsensusError::RoleViolation { - current_role: "Learner", - required_role: "Leader", - context: format!( - "Learner node {} receives RaftEvent::PromoteReadyLearners", - ctx.node_id - ), - } - .into()); - } - - RaftEvent::MembershipApplied => { - // Check if this learner has been promoted to Voter - let my_id = self.node_id(); - let node_meta = ctx.membership().retrieve_node_meta(my_id).await; - - if let Some(meta) = node_meta { - // Check if role is Voter (any role except LEARNER) - // FOLLOWER=0, CANDIDATE=1, LEADER=2, LEARNER=3 - let is_voter = meta.role != NodeRole::Learner as i32; - - if is_voter { - info!( - "Learner {} detected promotion to Voter (role={}), transitioning to Follower", - my_id, meta.role - ); - - // Transition to Follower role - role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send BecomeFollower event: {}", error_str); - NetworkError::SingalSendFailed(error_str) - })?; - } else { - debug!( - "Learner {} still in Learner role (role={}) after MembershipApplied", - my_id, meta.role - ); - } - } else { - warn!( - "Learner {} not found in membership after MembershipApplied", - my_id - ); - } - } - RaftEvent::FatalError { source, error } => { error!("[Learner] Fatal error from {}: {}", source, error); return Err(crate::Error::Fatal(format!( "Fatal error from {source}: {error}" ))); } - - RaftEvent::StepDownSelfRemoved => { - // Unreachable: handled at Raft level before reaching RoleState - unreachable!("StepDownSelfRemoved should be handled in Raft::run()"); - } } Ok(()) @@ -650,6 +524,139 @@ impl RaftRoleState for LearnerState { role_tx, ) } + + /// Trigger an independent snapshot on this role (Raft §7 — each server snapshots + /// independently). Called when `should_snapshot()` returns true after SM apply. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_create_snapshot( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Prevent duplicate snapshot creation + if self.snapshot_in_progress.load(Ordering::Acquire) { + info!("Snapshot creation already in progress. Skipping duplicate request."); + return Ok(()); + } + + self.snapshot_in_progress.store(true, Ordering::Release); + let state_machine_handler = ctx.state_machine_handler().clone(); + + // Use spawn to perform snapshot creation in the background + let role_tx = role_tx.clone(); + tokio::spawn(async move { + let result = state_machine_handler.create_snapshot().await; + info!("SnapshotCreated event will be processed in another event thread"); + if let Err(e) = role_tx.send(RoleEvent::SnapshotCreated(result)) { + error!("Follower failed to send snapshot creation result: {}", e); + } + }); + + Ok(()) + } + + /// Process the completed snapshot result (success or error). + /// Leader: schedules log purge up to `last_included`. + /// Follower/Learner: updates local snapshot path. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_snapshot_created( + &mut self, + result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, + ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Reset snapshot_in_progress flag + self.snapshot_in_progress.store(false, Ordering::SeqCst); + + // Per Raft §7: Learner independently purges logs after snapshot generation + match result { + Ok((metadata, _path)) => { + if let Some(last_included) = metadata.last_included { + info!(?last_included, "Learner snapshot created, purging logs"); + + // Learner independently decides to purge after snapshot + if self.can_purge_logs(self.last_purged_index, last_included) { + match ctx.purge_executor().execute_purge(last_included).await { + Ok(_) => { + self.last_purged_index = Some(last_included); + info!(?last_included, "Learner logs purged successfully"); + } + Err(e) => { + error!(?e, "Failed to purge logs after snapshot"); + } + } + } + } + } + Err(e) => { + error!(?e, "Learner snapshot creation failed"); + } + } + Ok(()) + } + + /// Membership config change applied to state — refresh any role-local derived state. + /// Leader: invalidates `cluster_metadata` cache. + /// Learner: checks if promoted to Voter; emits `BecomeFollower` if so. + /// Default: no-op (Follower/Candidate have no derived state to refresh). + async fn handle_membership_applied( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Check if this learner has been promoted to Voter + let my_id = self.node_id(); + let node_meta = ctx.membership().retrieve_node_meta(my_id).await; + + if let Some(meta) = node_meta { + // Check if role is Voter (any role except LEARNER) + // FOLLOWER=0, CANDIDATE=1, LEADER=2, LEARNER=3 + let is_voter = meta.role != NodeRole::Learner as i32; + + if is_voter { + info!( + "Learner {} detected promotion to Voter (role={}), transitioning to Follower", + my_id, meta.role + ); + + // Transition to Follower role + role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { + error!("Failed to send BecomeFollower event: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) + })?; + } else { + debug!( + "Learner {} still in Learner role (role={}) after MembershipApplied", + my_id, meta.role + ); + } + } else { + warn!( + "Learner {} not found in membership after MembershipApplied", + my_id + ); + } + Ok(()) + } + + // Stale in-flight events — these are Leader-only operations that may arrive + // after a step-down due to role_tx (unbounded, P2) race with BecomeFollower. + // Both orderings are valid; Learner silently ignores them rather than erroring. + + fn handle_log_purge_completed( + &mut self, + _purged_id: d_engine_proto::common::LogId, + ) -> Result<()> { + Ok(()) + } + + async fn handle_promote_ready_learners( + &mut self, + _ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + Ok(()) + } } impl LearnerState { diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index dd1247c6..fde74f08 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -1070,9 +1070,8 @@ async fn test_learner_promotion_on_membership_applied() { let mut state = LearnerState::::new(3, context.node_config.clone()); let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::MembershipApplied; - let result = state.handle_raft_event(raft_event, &context, role_tx).await; + let result = state.handle_membership_applied(&context, &role_tx).await; assert!(result.is_ok(), "MembershipApplied should succeed"); let role_event = tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()) @@ -1123,9 +1122,8 @@ async fn test_learner_stays_learner_on_membership_applied() { let mut state = LearnerState::::new(3, context.node_config.clone()); let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::MembershipApplied; - let result = state.handle_raft_event(raft_event, &context, role_tx).await; + let result = state.handle_membership_applied(&context, &role_tx).await; assert!(result.is_ok(), "MembershipApplied should succeed"); let timeout_result = @@ -1164,9 +1162,8 @@ async fn test_learner_node_not_found_on_membership_applied() { let mut state = LearnerState::::new(3, context.node_config.clone()); let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::MembershipApplied; - let result = state.handle_raft_event(raft_event, &context, role_tx).await; + let result = state.handle_membership_applied(&context, &role_tx).await; assert!( result.is_ok(), "MembershipApplied should succeed even when node not found" @@ -1181,195 +1178,155 @@ async fn test_learner_node_not_found_on_membership_applied() { } // ============================================================================ -// Role Violation Tests Module +// Snapshot Tests Module // ============================================================================ -mod role_violation_tests { +mod snapshot_tests { use super::*; + use std::sync::atomic::Ordering; - /// Test: LearnerState rejects leader-only events + /// Test: Learner gracefully ignores stale leader-only internal events /// - /// Scenario: - /// - Learner receives events only Leader can handle: - /// - CreateSnapshotEvent - /// - SnapshotCreated - /// - LogPurgeCompleted - /// - /// Expected: - /// - All return RoleViolation error + /// Protocol scenario: a leader steps down or a stale event arrives. + /// LogPurgeCompleted, PromoteReadyLearners, StepDownSelfRemoved must all be + /// silently ignored by a learner — they carry no meaning here and must not error. /// - /// Original: test_role_violation_events (in module) + /// Expected: all return Ok(()) — no panic, no state change. #[tokio::test] - async fn test_learner_rejects_leader_only_events() { + async fn test_learner_ignores_stale_leader_internal_events() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); - - // [Test LogPurgeCompleted] - // Learner now handles CreateSnapshotEvent and SnapshotCreated independently (Raft §7) - // but should NOT receive LogPurgeCompleted from external sources let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::LogPurgeCompleted(LogId { term: 1, index: 1 }); - let e = state.handle_raft_event(raft_event, &context, role_tx).await.unwrap_err(); assert!( - matches!( - e, - crate::Error::Consensus(crate::ConsensusError::RoleViolation { .. }) - ), - "LogPurgeCompleted should return RoleViolation" + state.handle_log_purge_completed(LogId { term: 1, index: 1 }).is_ok(), + "Stale LogPurgeCompleted should be silently ignored" + ); + assert!( + state.handle_promote_ready_learners(&context, &role_tx).await.is_ok(), + "Stale PromoteReadyLearners should be silently ignored" + ); + assert!( + state.handle_self_removed(&role_tx).is_ok(), + "Stale StepDownSelfRemoved should be silently ignored" ); } - /// Test: Learner ignores duplicate CreateSnapshotEvent while snapshot is in progress - /// - /// Purpose: - /// Validates that the snapshot_in_progress flag prevents concurrent snapshot creation, - /// ensuring snapshot consistency and avoiding resource waste from duplicate operations. + /// Test: Learner ignores duplicate CreateSnapshot while one is already in progress /// - /// Scenario: - /// - First CreateSnapshotEvent is received and sets snapshot_in_progress=true - /// - Second CreateSnapshotEvent arrives before first completes + /// The `snapshot_in_progress` flag guards against concurrent snapshot creation. /// /// Expected: - /// - First event: Returns Ok(), starts async snapshot creation - /// - Second event: Returns Ok() immediately without starting new snapshot (logged as skipped) - /// - snapshot_in_progress flag protects against concurrent creation + /// - First call: Ok(), sets snapshot_in_progress = true, spawns background task + /// - Second call: Ok(), skips (flag already set), flag remains true #[tokio::test] async fn test_learner_ignores_duplicate_create_snapshot_event() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); - - // First CreateSnapshotEvent - should succeed let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result1 = state - .handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx.clone()) - .await; - assert!(result1.is_ok(), "First CreateSnapshotEvent should succeed"); - // Verify flag is set + // First trigger — starts background snapshot + let result1 = state.handle_create_snapshot(&context, &role_tx).await; + assert!(result1.is_ok(), "First CreateSnapshotEvent should succeed"); assert!( - state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), + state.snapshot_in_progress.load(Ordering::SeqCst), "snapshot_in_progress should be true after first event" ); - // Second CreateSnapshotEvent - should be ignored - let result2 = - state.handle_raft_event(RaftEvent::CreateSnapshotEvent, &context, role_tx).await; + // Second trigger while first is still running — must be a no-op + let result2 = state.handle_create_snapshot(&context, &role_tx).await; assert!( result2.is_ok(), "Second CreateSnapshotEvent should return Ok (ignored)" ); - - // Flag should still be true (first snapshot still in progress) assert!( - state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), + state.snapshot_in_progress.load(Ordering::SeqCst), "snapshot_in_progress should remain true" ); } - /// Test: Learner resets snapshot_in_progress flag after SnapshotCreated (success case) + /// Test: Learner resets snapshot_in_progress and updates last_purged_index on success /// - /// Purpose: - /// Validates that the snapshot_in_progress flag is correctly reset after snapshot completion, - /// allowing subsequent snapshots to be created when needed. + /// Per Raft §7, learners independently purge logs after a successful snapshot. /// /// Scenario: - /// - snapshot_in_progress is manually set to true (simulating ongoing snapshot) - /// - SnapshotCreated event with successful result is received + /// - snapshot_in_progress pre-set to true (simulating in-flight snapshot) + /// - SnapshotCreated arrives with successful result (last_included = index 50) /// /// Expected: - /// - Event handler returns Ok() - /// - snapshot_in_progress flag is reset to false - /// - System is ready to accept new CreateSnapshotEvent - /// - /// This ensures the flag lifecycle is: false → true (on create) → false (on complete) + /// - snapshot_in_progress reset to false + /// - last_purged_index updated to Some(LogId { term: 1, index: 50 }) #[tokio::test] async fn test_learner_resets_snapshot_flag_on_success() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); + state.snapshot_in_progress.store(true, Ordering::SeqCst); + // Prerequisite: commit_index must exceed last_included for can_purge_logs to allow purge. + // In real operation, entries are committed before they can be snapshotted. + state.update_commit_index(100).unwrap(); - // Simulate snapshot in progress - state.snapshot_in_progress.store(true, std::sync::atomic::Ordering::SeqCst); - - // Create successful snapshot result + let last_included = LogId { term: 1, index: 50 }; let metadata = d_engine_proto::server::storage::SnapshotMetadata { - last_included: Some(LogId { term: 1, index: 50 }), + last_included: Some(last_included), checksum: bytes::Bytes::new(), }; let snapshot_result = Ok((metadata, std::path::PathBuf::from("/tmp/test_snapshot.bin"))); let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state - .handle_raft_event( - RaftEvent::SnapshotCreated(snapshot_result), - &context, - role_tx, - ) - .await; + let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; assert!(result.is_ok(), "SnapshotCreated should succeed"); - - // Verify flag is reset assert!( - !state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), + !state.snapshot_in_progress.load(Ordering::SeqCst), "snapshot_in_progress should be false after SnapshotCreated" ); + assert_eq!( + state.last_purged_index, + Some(last_included), + "last_purged_index must advance to last_included after log purge" + ); } - /// Test: Learner resets snapshot_in_progress flag after SnapshotCreated (failure case) - /// - /// Purpose: - /// Validates that the snapshot_in_progress flag is reset even when snapshot creation fails, - /// allowing the system to retry snapshot creation later without being permanently blocked. + /// Test: Learner resets snapshot_in_progress on failure but does NOT purge logs /// - /// Scenario: - /// - snapshot_in_progress is set to true - /// - SnapshotCreated event with error result is received + /// A failed snapshot must not advance the purge boundary. + /// The flag must still clear so the next ApplyCompleted can retry. /// /// Expected: - /// - Event handler returns Ok() (error is logged but not propagated) - /// - snapshot_in_progress flag is reset to false - /// - System can retry snapshot creation on next ApplyCompleted trigger - /// - /// This ensures failure recovery: the flag doesn't stay locked after an error + /// - snapshot_in_progress reset to false + /// - last_purged_index remains None + /// - handler returns Ok() (error logged, not propagated) #[tokio::test] async fn test_learner_resets_snapshot_flag_on_failure() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); + state.snapshot_in_progress.store(true, Ordering::SeqCst); - // Simulate snapshot in progress - state.snapshot_in_progress.store(true, std::sync::atomic::Ordering::SeqCst); - - // Create failed snapshot result let snapshot_result = Err(crate::Error::Fatal("Snapshot creation failed".to_string())); let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state - .handle_raft_event( - RaftEvent::SnapshotCreated(snapshot_result), - &context, - role_tx, - ) - .await; + let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; assert!( result.is_ok(), "SnapshotCreated with error should return Ok" ); - - // Verify flag is reset even on failure assert!( - !state.snapshot_in_progress.load(std::sync::atomic::Ordering::SeqCst), + !state.snapshot_in_progress.load(Ordering::SeqCst), "snapshot_in_progress should be false after failed SnapshotCreated" ); + assert_eq!( + state.last_purged_index, None, + "last_purged_index must not advance when snapshot failed" + ); } } @@ -1539,8 +1496,8 @@ async fn test_learner_serves_eventual_read_locally() { /// - State machine handler indicates snapshot should be taken /// /// Expected: -/// - CreateSnapshotEvent is sent back to role event loop for processing -/// - Event is reprocessed as RoleEvent::ReprocessEvent +/// - RoleEvent::CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) +/// - No ReprocessEvent wrapper — direct send eliminates the bounded event_tx deadlock path #[tokio::test] async fn test_apply_completed_triggers_snapshot_when_condition_met() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1575,19 +1532,12 @@ async fn test_apply_completed_triggers_snapshot_when_condition_met() { "ApplyCompleted should be handled successfully, got: {result:?}" ); - // VERIFY 2: CreateSnapshotEvent is sent as RoleEvent::ReprocessEvent - let event = role_rx.try_recv().expect("Should receive snapshot event"); - match event { - RoleEvent::ReprocessEvent(boxed_event) => { - match *boxed_event { - RaftEvent::CreateSnapshotEvent => { - // Success! Event is correctly wrapped - } - other => panic!("Expected CreateSnapshotEvent, got: {other:?}"), - } - } - other => panic!("Expected RoleEvent::ReprocessEvent, got: {other:?}"), - } + // VERIFY 2: CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) + let event = role_rx.try_recv().expect("Should receive snapshot trigger event"); + assert!( + matches!(event, RoleEvent::CreateSnapshotEvent), + "Expected RoleEvent::CreateSnapshotEvent, got: {event:?}" + ); // VERIFY 3: No additional events queued assert!( diff --git a/d-engine-core/src/raft_role/mod.rs b/d-engine-core/src/raft_role/mod.rs index 5ed47b96..8e8f3a9f 100644 --- a/d-engine-core/src/raft_role/mod.rs +++ b/d-engine-core/src/raft_role/mod.rs @@ -25,7 +25,9 @@ use std::sync::atomic::Ordering; use candidate_state::CandidateState; use d_engine_proto::client::ClientReadRequest; +use d_engine_proto::common::LogId; use d_engine_proto::server::election::VotedFor; +use d_engine_proto::server::storage::SnapshotMetadata; use follower_state::FollowerState; pub use leader_state::ClusterMetadata; use leader_state::LeaderState; @@ -501,6 +503,73 @@ impl RaftRole { ) -> crate::Result<()> { self.state_mut().handle_append_result(follower_id, result, ctx, role_tx).await } + + /// Trigger an independent snapshot on this role (Raft §7 — each server snapshots + /// independently). Called when `should_snapshot()` returns true after SM apply. + /// Candidate: returns `RoleViolation` — transient state, defer to next stable role. + pub(crate) async fn handle_create_snapshot( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + self.state_mut().handle_create_snapshot(ctx, role_tx).await + } + + /// Process the completed snapshot result (success or error). + /// Leader: schedules log purge up to `last_included`. + /// Follower/Learner: updates local snapshot path. + /// Candidate: returns `RoleViolation`. + pub(crate) async fn handle_snapshot_created( + &mut self, + result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + self.state_mut().handle_snapshot_created(result, ctx, role_tx).await + } + + /// Advance the purge boundary after log entries up to `purged_id` are removed. + /// Leader only: updates `last_purged_index`. + /// Non-leader: no-op (unexpected; logs a warning). + pub(crate) fn handle_log_purge_completed( + &mut self, + purged_id: LogId, + ) -> Result<()> { + self.state_mut().handle_log_purge_completed(purged_id) + } + + /// Check pending learners for promotion eligibility after a membership change. + /// Leader only: evaluates `pending_promotions`, proposes config change if ready. + /// Non-leader: no-op (logs a warning — unexpected in steady state). + pub(crate) async fn handle_promote_ready_learners( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + self.state_mut().handle_promote_ready_learners(ctx, role_tx).await + } + + /// Node was removed from cluster membership; step down immediately per Raft protocol. + /// Leader: emits `BecomeFollower`. Non-leader: unreachable (only Leader proposes + /// self-removal via config change). + pub(crate) fn handle_self_removed( + &mut self, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + self.state_mut().handle_self_removed(role_tx) + } + + /// Membership config change applied to state — refresh any role-local derived state. + /// Leader: invalidates `cluster_metadata` cache for hot-path reads. + /// Learner: checks if it was promoted to Voter; emits `BecomeFollower` if so. + /// Follower/Candidate: no-op. + pub(crate) async fn handle_membership_applied( + &mut self, + ctx: &RaftContext, + role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + self.state_mut().handle_membership_applied(ctx, role_tx).await + } } impl Serialize for HardState { diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index 2e7f98a5..67601446 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -1,14 +1,17 @@ +use crate::ConsensusError; use crate::client::{ClientReadRequest, ClientResponse, KvEntry, LeaderHint}; use async_trait::async_trait; +use d_engine_proto::common::LogId; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::replication::AppendEntriesRequest; use d_engine_proto::server::replication::AppendEntriesResponse; +use d_engine_proto::server::storage::SnapshotMetadata; use tokio::sync::mpsc; use tokio::time::Instant; use tonic::Status; -use tracing::debug; use tracing::error; use tracing::warn; +use tracing::{debug, info}; use super::RaftRole; use super::SharedState; @@ -229,9 +232,8 @@ pub trait RaftRoleState: Send + Sync + 'static { current_term, })) .map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send NotifyNewCommitIndex: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send NotifyNewCommitIndex: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) })?; Ok(()) @@ -504,9 +506,8 @@ pub trait RaftRoleState: Send + Sync + 'static { if is_new_leader { role_tx.send(RoleEvent::LeaderDiscovered(new_leader_id, request_term)).map_err( |e| { - let error_str = format!("{e:?}"); - error!("Failed to send LeaderDiscovered: {}", error_str); - NetworkError::SingalSendFailed(error_str) + error!("Failed to send LeaderDiscovered: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) }, )?; } @@ -586,6 +587,111 @@ pub trait RaftRoleState: Send + Sync + 'static { // No-op for non-leader roles (only Leader has read buffer to drain) Err(MembershipError::NotLeader.into()) } + + /// Trigger an independent snapshot on this role (Raft §7 — each server snapshots + /// independently). Called when `should_snapshot()` returns true after SM apply. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_create_snapshot( + &mut self, + _ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + Err(ConsensusError::RoleViolation { + current_role: "Candidate", + required_role: "Follower/Leader/Learner", + context: ("Candidate node attempted to create snapshot.").to_string(), + } + .into()) + } + + /// Process the completed snapshot result (success or error). + /// Leader: schedules log purge up to `last_included`. + /// Follower/Learner: updates local snapshot path. + /// Default: no-op. Candidate overrides to return `RoleViolation`. + async fn handle_snapshot_created( + &mut self, + _result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, + _ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + Err(ConsensusError::RoleViolation { + current_role: "Candidate", + required_role: "Follower/Leader/Learner", + context: ("Candidate role node attempted to handle created snapshot.").to_string(), + } + .into()) + } + + /// Advance the purge boundary after log entries up to `purged_id` are removed. + /// Leader only: updates `last_purged_index`. + /// Default: no-op + warn (unexpected on non-leader). + fn handle_log_purge_completed( + &mut self, + _purged_id: LogId, + ) -> Result<()> { + Err(ConsensusError::RoleViolation { + current_role: "non-Leader", + required_role: "Leader", + context: "LogPurgeCompleted is a Leader-only event.".to_string(), + } + .into()) + } + + /// Check pending learners for promotion eligibility after a membership change. + /// Leader only: evaluates `pending_promotions`, proposes config change if ready. + /// Default: no-op + warn (unexpected on non-leader). + async fn handle_promote_ready_learners( + &mut self, + ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + Err(ConsensusError::RoleViolation { + current_role: "None Leader", + required_role: "Leader", + context: format!( + "None Leader node {} receives RaftEvent::PromoteReadyLearners", + ctx.node_id + ), + } + .into()) + } + + /// Node was removed from cluster membership; step down immediately per Raft protocol. + /// Leader: emits `BecomeFollower`. Non-leader: unreachable in practice. + /// Default: warn + Ok(()) (defensive; only Leader should receive this). + fn handle_self_removed( + &mut self, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + warn!( + "Node {} received StepDownSelfRemoved in non-leader role — ignoring", + self.node_id() + ); + Ok(()) + } + + /// Membership config change applied to state — refresh any role-local derived state. + /// Leader: invalidates `cluster_metadata` cache. + /// Learner: checks if promoted to Voter; emits `BecomeFollower` if so. + /// Default: no-op (Follower/Candidate have no derived state to refresh). + async fn handle_membership_applied( + &mut self, + ctx: &RaftContext, + _role_tx: &mpsc::UnboundedSender, + ) -> Result<()> { + // Followers don't maintain cluster metadata cache + // This event is only relevant for leaders + info!("Follower/Candidate node ignoring MembershipApplied event"); + Err(ConsensusError::RoleViolation { + current_role: "None Leader/Learner", + required_role: "Leader/Learner", + context: format!( + "None Leader/Learner node {} receives RaftEvent::PromoteReadyLearners", + ctx.node_id + ), + } + .into()) + } } /// Send a RaftEvent back into the role event loop for reprocessing. @@ -594,9 +700,8 @@ pub(super) fn send_replay_raft_event( raft_event: RaftEvent, ) -> Result<()> { role_tx.send(RoleEvent::ReprocessEvent(Box::new(raft_event))).map_err(|e| { - let error_str = format!("{e:?}"); - error!("Failed to send: {}", error_str); - NetworkError::SingalSendFailed(error_str).into() + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)).into() }) } @@ -618,7 +723,10 @@ pub(super) fn check_and_trigger_snapshot( current_term, }) { - send_replay_raft_event(role_tx, RaftEvent::CreateSnapshotEvent)?; + role_tx.send(RoleEvent::CreateSnapshotEvent).map_err(|e| { + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) + })?; } Ok(()) } diff --git a/d-engine-core/src/raft_test/mod.rs b/d-engine-core/src/raft_test/mod.rs index 2537adac..1012f627 100644 --- a/d-engine-core/src/raft_test/mod.rs +++ b/d-engine-core/src/raft_test/mod.rs @@ -98,6 +98,25 @@ //! #### B7. ReprocessEvent Event //! - **B7.1**: Event re-queued to event_rx for reprocessing //! +//! #### B8. CreateSnapshotEvent (migrated from P4 event_tx → P2 role_tx) +//! - **B8.1**: Follower — dispatch returns Ok, snapshot task spawned in background +//! - **B8.2**: Candidate — RoleViolation is non-fatal, swallowed by dispatch layer +//! +//! #### B9. SnapshotCreated (migrated from P4 event_tx → P2 role_tx) +//! - **B9.1**: Follower + Err payload — dispatch returns Ok (non-fatal error swallowed) +//! +//! #### B10. StepDownSelfRemoved (migrated from P4 event_tx → P2 role_tx) +//! - **B10.1**: Leader — handle_self_removed enqueues BecomeFollower; role transitions to Follower +//! +//! #### B11. MembershipApplied (migrated from P4 event_tx → P2 role_tx) +//! - **B11.1**: Follower — RoleViolation swallowed; leader-specific logic in membership_change_test +//! +//! #### B12. PromoteReadyLearners (migrated from P4 event_tx → P2 role_tx) +//! - **B12.1**: Follower — RoleViolation swallowed; leader-specific logic in membership_change_test +//! +//! #### B13. LogPurgeCompleted (migrated from P4 event_tx → P2 role_tx) +//! - **B13.1**: Follower — dispatch returns Ok, last_purged_index updated +//! //! ### C. Leader Initialization Tests (Peer State Setup) //! //! These tests verify that newly elected leaders correctly initialize peer state diff --git a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs index 364301cb..e2babf0e 100644 --- a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs +++ b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs @@ -1061,6 +1061,167 @@ async fn test_reprocess_event_requeues() { assert!(is_follower(raft.role.as_i32())); } +// B8–B13. New Lifecycle Event Dispatch Tests +// +// Six RoleEvent variants were migrated from bounded event_tx (P4) to unbounded +// role_tx (P2) to eliminate deadlock when the P4 channel is saturated by inbound RPCs. +// These tests verify that each dispatch arm in handle_role_event routes correctly +// — no todo!() panic, correct return value, and the observable role-level outcome. +// +// Handler-level behaviour (flag mutations, purge logic, metadata refresh) is already +// covered in each role's unit-test module; these tests focus only on the routing layer. + +/// B8.1 — CreateSnapshotEvent dispatched to Follower +/// +/// Follower's handle_create_snapshot spawns a background snapshot task and returns Ok. +/// The dispatch must succeed without panicking or propagating a fatal error. +#[tokio::test] +async fn test_create_snapshot_event_dispatch_on_follower() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + assert!(is_follower(raft.role.as_i32())); + + raft.handle_role_event(RoleEvent::CreateSnapshotEvent) + .await + .expect("CreateSnapshotEvent on Follower must not return a fatal error"); + + // CreateSnapshotEvent is a background trigger — role is unchanged. + assert!(is_follower(raft.role.as_i32())); +} + +/// B8.2 — CreateSnapshotEvent dispatched to Candidate +/// +/// Candidate rejects snapshot creation with RoleViolation (non-fatal). +/// The dispatch layer swallows non-fatal errors and returns Ok to the caller. +#[tokio::test] +async fn test_create_snapshot_event_dispatch_on_candidate() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + assert!(is_candidate(raft.role.as_i32())); + + // RoleViolation is non-fatal — dispatch swallows it and returns Ok. + raft.handle_role_event(RoleEvent::CreateSnapshotEvent) + .await + .expect("CreateSnapshotEvent on Candidate must return Ok (RoleViolation swallowed)"); + + assert!( + is_candidate(raft.role.as_i32()), + "Role must remain Candidate" + ); +} + +/// B9 — SnapshotCreated(Err) dispatched to Follower +/// +/// When snapshot creation fails, SnapshotCreated carries an Err payload. +/// The handler resets snapshot_in_progress and returns a non-fatal error. +/// The dispatch layer swallows it and returns Ok — role is unchanged. +#[tokio::test] +async fn test_snapshot_created_err_dispatch_on_follower() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + // Use a non-fatal snapshot error so the dispatch does not propagate it. + let err_result = Err(crate::SnapshotError::OperationFailed("simulated failure".into()).into()); + + raft.handle_role_event(RoleEvent::SnapshotCreated(err_result)) + .await + .expect("SnapshotCreated(Err) on Follower must return Ok (non-fatal error swallowed)"); + + assert!(is_follower(raft.role.as_i32())); +} + +/// B10 — StepDownSelfRemoved on Leader causes role transition to Follower +/// +/// This is the most behaviourally significant of the six new events at the integration +/// level. handle_self_removed posts BecomeFollower(None) onto role_tx so the main loop +/// can process the transition on the next iteration. In unit tests we drive that queued +/// event manually to verify the complete Leader → StepDownSelfRemoved → Follower path. +#[tokio::test] +async fn test_step_down_self_removed_leader_transitions_to_follower() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + let (raft_log, replication_core) = prepare_succeed_majority_confirmation(); + raft.ctx.storage.raft_log = Arc::new(raft_log); + raft.ctx.handlers.replication_handler = replication_core; + + // Establish Leader role. + raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + assert!(is_leader(raft.role.as_i32())); + + // Step 1: dispatch StepDownSelfRemoved — handle_self_removed enqueues + // BecomeFollower(None) on the internal role_tx and returns Ok. + raft.handle_role_event(RoleEvent::StepDownSelfRemoved) + .await + .expect("StepDownSelfRemoved on Leader must not return a fatal error"); + + // Step 2: simulate the main loop consuming the queued BecomeFollower(None). + // In production this is driven by the biased select! loop; in unit tests + // we drive it manually to verify the full transition completes. + raft.handle_role_event(RoleEvent::BecomeFollower(None)) + .await + .expect("BecomeFollower queued by StepDownSelfRemoved must succeed"); + + assert!( + is_follower(raft.role.as_i32()), + "Leader must step down to Follower after self-removal" + ); +} + +/// B11 — MembershipApplied dispatched to Follower +/// +/// MembershipApplied is meaningful only for the Leader (it refreshes its metadata cache). +/// Follower returns RoleViolation (non-fatal); the dispatch layer swallows it. +/// The leader-specific behaviour is already covered in membership_change_test.rs. +#[tokio::test] +async fn test_membership_applied_dispatch_on_follower() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_role_event(RoleEvent::MembershipApplied) + .await + .expect("MembershipApplied on Follower must return Ok (RoleViolation swallowed)"); + + assert!(is_follower(raft.role.as_i32())); +} + +/// B12 — PromoteReadyLearners dispatched to Follower +/// +/// Only the Leader evaluates learner promotion eligibility. +/// Follower returns RoleViolation (non-fatal); the dispatch layer swallows it. +/// The leader-specific promotion logic is already covered in membership_change_test.rs. +#[tokio::test] +async fn test_promote_ready_learners_dispatch_on_follower() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_role_event(RoleEvent::PromoteReadyLearners) + .await + .expect("PromoteReadyLearners on Follower must return Ok (RoleViolation swallowed)"); + + assert!(is_follower(raft.role.as_i32())); +} + +/// B13 — LogPurgeCompleted dispatched to Follower +/// +/// All roles update their internal last_purged_index on LogPurgeCompleted. +/// This test confirms the dispatch arm routes correctly and returns Ok without panicking. +#[tokio::test] +async fn test_log_purge_completed_dispatch_on_follower() { + use d_engine_proto::common::LogId; + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_role_event(RoleEvent::LogPurgeCompleted(LogId { term: 1, index: 50 })) + .await + .expect("LogPurgeCompleted on Follower must not return a fatal error"); + + assert!(is_follower(raft.role.as_i32())); +} + // ============================================================================ // C. LEADER INITIALIZATION TESTS (Peer State Setup) // ============================================================================ diff --git a/d-engine-proto/proto/server/storage.proto b/d-engine-proto/proto/server/storage.proto index 9bc2ce56..41a41437 100644 --- a/d-engine-proto/proto/server/storage.proto +++ b/d-engine-proto/proto/server/storage.proto @@ -50,11 +50,12 @@ message SnapshotAck { uint32 next_requested = 3; enum ChunkStatus { - ACCEPTED = 0; - CHECKSUM_MISMATCH = 1; - OUT_OF_ORDER = 2; - REQUESTED = 3; - FAILED = 4; + CHUNK_STATUS_UNSPECIFIED = 0; + CHUNK_STATUS_ACCEPTED = 1; + CHUNK_STATUS_CHECKSUM_MISMATCH = 2; + CHUNK_STATUS_OUT_OF_ORDER = 3; + CHUNK_STATUS_REQUESTED = 4; + CHUNK_STATUS_FAILED = 5; } } diff --git a/d-engine-proto/src/generated/d_engine.server.storage.rs b/d-engine-proto/src/generated/d_engine.server.storage.rs index af3069c6..054c7ee3 100644 --- a/d-engine-proto/src/generated/d_engine.server.storage.rs +++ b/d-engine-proto/src/generated/d_engine.server.storage.rs @@ -76,11 +76,12 @@ pub mod snapshot_ack { )] #[repr(i32)] pub enum ChunkStatus { - Accepted = 0, - ChecksumMismatch = 1, - OutOfOrder = 2, - Requested = 3, - Failed = 4, + Unspecified = 0, + Accepted = 1, + ChecksumMismatch = 2, + OutOfOrder = 3, + Requested = 4, + Failed = 5, } impl ChunkStatus { /// String value of the enum field names used in the ProtoBuf definition. @@ -89,21 +90,23 @@ pub mod snapshot_ack { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Self::Accepted => "ACCEPTED", - Self::ChecksumMismatch => "CHECKSUM_MISMATCH", - Self::OutOfOrder => "OUT_OF_ORDER", - Self::Requested => "REQUESTED", - Self::Failed => "FAILED", + Self::Unspecified => "CHUNK_STATUS_UNSPECIFIED", + Self::Accepted => "CHUNK_STATUS_ACCEPTED", + Self::ChecksumMismatch => "CHUNK_STATUS_CHECKSUM_MISMATCH", + Self::OutOfOrder => "CHUNK_STATUS_OUT_OF_ORDER", + Self::Requested => "CHUNK_STATUS_REQUESTED", + Self::Failed => "CHUNK_STATUS_FAILED", } } /// Creates an enum from field names used in the ProtoBuf definition. pub fn from_str_name(value: &str) -> ::core::option::Option { match value { - "ACCEPTED" => Some(Self::Accepted), - "CHECKSUM_MISMATCH" => Some(Self::ChecksumMismatch), - "OUT_OF_ORDER" => Some(Self::OutOfOrder), - "REQUESTED" => Some(Self::Requested), - "FAILED" => Some(Self::Failed), + "CHUNK_STATUS_UNSPECIFIED" => Some(Self::Unspecified), + "CHUNK_STATUS_ACCEPTED" => Some(Self::Accepted), + "CHUNK_STATUS_CHECKSUM_MISMATCH" => Some(Self::ChecksumMismatch), + "CHUNK_STATUS_OUT_OF_ORDER" => Some(Self::OutOfOrder), + "CHUNK_STATUS_REQUESTED" => Some(Self::Requested), + "CHUNK_STATUS_FAILED" => Some(Self::Failed), _ => None, } } diff --git a/d-engine-server/src/node/builder.rs b/d-engine-server/src/node/builder.rs index 46d89b65..220f59aa 100644 --- a/d-engine-server/src/node/builder.rs +++ b/d-engine-server/src/node/builder.rs @@ -427,7 +427,7 @@ where // role_tx / role_rx created earlier (before raft_log) to pass log_flush_tx to start(). let (event_tx, event_rx) = mpsc::channel(10240); let (cmd_tx, cmd_rx) = mpsc::channel(node_config.raft.cmd_channel_capacity); - let event_tx_clone = event_tx.clone(); // used in commit handler + let role_tx_clone = role_tx.clone(); let role_tx_for_sm = role_tx.clone(); // used in SM worker (ApplyCompleted → P2) // Bridge zombie signals from health monitor → role event loop. @@ -548,7 +548,7 @@ where state_machine_handler, raft_log: raft_core.ctx.storage.raft_log.clone(), membership: membership.clone(), - event_tx: event_tx_clone, + role_tx: role_tx_clone, sm_apply_tx, shutdown_signal, max_batch_size: raft_core.ctx.node_config.raft.batching.max_batch_size, From e1f81555cb8d1e2a6b3cbdfdf0400d44aa3e134a Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:50:17 +0800 Subject: [PATCH 2/4] refactor #413: Rename RaftEvent to InboundEvent and rename RoleEvent to InternalEvent --- CHANGELOG.md | 4 +- README.md | 2 +- d-engine-core/benches/leader_state_bench.rs | 16 +- .../commit_handler/default_commit_handler.rs | 16 +- .../default_commit_handler_test.rs | 73 ++-- d-engine-core/src/config/raft.rs | 2 +- d-engine-core/src/event.rs | 36 +- d-engine-core/src/membership.rs | 2 +- d-engine-core/src/network/mod.rs | 4 +- d-engine-core/src/raft.rs | 294 +++++++------ .../src/raft_role/candidate_state.rs | 91 ++-- .../src/raft_role/candidate_state_test.rs | 262 ++++++----- d-engine-core/src/raft_role/follower_state.rs | 64 +-- .../src/raft_role/follower_state_test.rs | 411 +++++++++++------- d-engine-core/src/raft_role/leader_state.rs | 285 ++++++------ .../leader_state_test/become_follower_test.rs | 72 +-- .../leader_state_test/buffer_cleanup_test.rs | 8 +- .../leader_state_test/client_read_test.rs | 114 +++-- .../leader_state_test/client_write_test.rs | 115 +++-- .../leader_state_test/commit_index_test.rs | 90 ++-- .../leader_state_test/deadline_test.rs | 60 +-- .../leader_state_test/event_handling_test.rs | 213 +++++---- .../leader_state_test/fatal_error_test.rs | 32 +- .../lease_refresh_on_log_flushed_test.rs | 42 +- .../membership_change_test.rs | 153 ++++--- .../pending_commit_actions_test.rs | 65 +-- .../pending_lease_reads_test.rs | 68 +-- .../leader_state_test/pending_reads_test.rs | 55 ++- .../leader_state_test/replication_test.rs | 239 ++++++---- .../single_voter_commit_test.rs | 20 +- .../leader_state_test/snapshot_test.rs | 34 +- .../leader_state_test/snapshot_worker_test.rs | 28 +- .../stale_learner_deadline_test.rs | 27 +- .../state_management_test.rs | 26 +- .../worker_lifecycle_test.rs | 34 +- d-engine-core/src/raft_role/learner_state.rs | 66 +-- .../src/raft_role/learner_state_test.rs | 301 +++++++------ d-engine-core/src/raft_role/mod.rs | 68 +-- d-engine-core/src/raft_role/raft_role_test.rs | 26 +- d-engine-core/src/raft_role/role_state.rs | 82 ++-- .../drain_based_batch_architecture_tests.rs | 44 +- .../src/raft_test/leader_discovered_tests.rs | 22 +- .../raft_test/merge_append_entries_tests.rs | 221 +++++----- d-engine-core/src/raft_test/mod.rs | 54 +-- ...sts.rs => process_inbound_events_tests.rs} | 48 +- .../src/raft_test/raft_comprehensive_tests.rs | 333 +++++++------- d-engine-core/src/replication/mod.rs | 2 +- .../src/state_machine_handler/worker.rs | 30 +- .../src/state_machine_handler/worker_test.rs | 74 ++-- .../src/storage/buffered_raft_log.rs | 16 +- .../buffered_raft_log_test/term_index_test.rs | 2 +- .../src/test_utils/mock/mock_raft_builder.rs | 30 +- d-engine-server/src/api/embedded_client.rs | 8 +- .../src/membership/raft_membership.rs | 4 +- .../src/network/grpc/grpc_raft_service.rs | 28 +- .../network/grpc/grpc_raft_service_test.rs | 16 +- d-engine-server/src/network/health_monitor.rs | 6 +- .../src/network/health_monitor_test.rs | 2 +- d-engine-server/src/node/builder.rs | 31 +- d-engine-server/src/node/mod.rs | 6 +- .../adaptors/rocksdb/rocksdb_state_machine.rs | 4 +- .../src/test_utils/mock/mock_node_builder.rs | 30 +- .../select_fairness_embedded.rs | 4 +- .../embedded_client_operations.rs | 2 +- .../docs/server_guide/snapshot-guarantees.md | 20 +- 65 files changed, 2600 insertions(+), 2037 deletions(-) rename d-engine-core/src/raft_test/{process_raft_events_tests.rs => process_inbound_events_tests.rs} (88%) diff --git a/CHANGELOG.md b/CHANGELOG.md index d312d1eb..3bf0fdf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ All notable changes to this project will be documented in this file. - **`scan_prefix` API** (#378): `ClientApi::scan_prefix(prefix)` returns all KV pairs matching a prefix in a single read — intended for zero-race-window state re-sync after watch reconnection. - **Async IO architecture — pipeline replication** (#334, #341, #342, #343, #345, #349, #350, #351): - The Raft event loop is now fully non-blocking under write load + The Inbound event loop is now fully non-blocking under write load - `BufferedRaftLog` runs on a dedicated OS thread — WAL writes never steal tokio worker threads - `StateMachineWorker::apply_batch` is fully async — RocksDB apply no longer blocks the event loop - AppendEntries uses a **persistent bidirectional gRPC stream per peer** — eliminates per-batch connection overhead @@ -78,7 +78,7 @@ All notable changes to this project will be documented in this file. - Eliminates election churn when recovering or newly-joined nodes have lagged logs - Aligns with Raft §5.2 (term check takes priority over log matching) -- **fix(snapshot) #315**: Snapshot disk I/O isolated from Raft event loop via `spawn_blocking` +- **fix(snapshot) #315**: Snapshot disk I/O isolated from Inbound event loop via `spawn_blocking` - Previously, snapshot transfer competed with consensus and caused `ProposeFailed` (4006) under load - **fix(snapshot) #308**: Snapshot install success is now driven by the follower's apply confirmation, not the transfer ACK diff --git a/README.md b/README.md index b6a5f51a..1b4ab742 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ and pluggable storage backends. Start with one node, scale to a cluster when nee ## New in v0.2.4 🎉 -- **Async IO Architecture**: Raft event loop is fully non-blocking — WAL writes, state machine apply, and replication all run off the hot path. AppendEntries uses a persistent bidirectional stream per peer; replication is pipelined across followers. +- **Async IO Architecture**: Inbound event loop is fully non-blocking — WAL writes, state machine apply, and replication all run off the hot path. AppendEntries uses a persistent bidirectional stream per peer; replication is pipelined across followers. - **Cluster Membership Streaming**: `EmbeddedEngine::watch_membership()` / `GrpcClient::watch_membership()` — subscribe to real-time membership changes in both embedded and standalone modes - **Simpler Startup**: `EmbeddedEngine::start(data_dir)` and `StandaloneEngine::run(data_dir, shutdown_rx)` — no config file required for common cases - **Jepsen Validated**: 5 workloads + 6-hour soak test under combined kill/partition/pause faults — see [Correctness Guarantees](https://github.com/deventlab/d-engine-jepsen/blob/main/GUARANTEES.md) diff --git a/d-engine-core/benches/leader_state_bench.rs b/d-engine-core/benches/leader_state_bench.rs index d690b869..873c0a04 100644 --- a/d-engine-core/benches/leader_state_bench.rs +++ b/d-engine-core/benches/leader_state_bench.rs @@ -150,7 +150,7 @@ struct BenchFixture { leader_state: LeaderState, raft_context: RaftContext, #[allow(dead_code)] - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, } impl BenchFixture { @@ -257,13 +257,13 @@ impl BenchFixture { node_config: Arc::new(node_config.clone()), }; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let leader_state = LeaderState::new(1, Arc::new(node_config)); BenchFixture { leader_state, raft_context, - role_tx, + internal_event_tx, } } } @@ -287,7 +287,10 @@ fn bench_process_promotions_2_nodes(c: &mut Criterion) { black_box( fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners( + &fixture.raft_context, + &fixture.internal_event_tx, + ) .await, ) }) @@ -320,7 +323,10 @@ fn bench_batch_promotion_scaling(c: &mut Criterion) { black_box( fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners( + &fixture.raft_context, + &fixture.internal_event_tx, + ) .await, ) }) diff --git a/d-engine-core/src/commit_handler/default_commit_handler.rs b/d-engine-core/src/commit_handler/default_commit_handler.rs index ca4dfb4f..36a8901d 100644 --- a/d-engine-core/src/commit_handler/default_commit_handler.rs +++ b/d-engine-core/src/commit_handler/default_commit_handler.rs @@ -11,11 +11,11 @@ use tracing::trace; use tracing::warn; use super::CommitHandler; +use crate::InternalEvent; use crate::Membership; use crate::NewCommitData; use crate::RaftLog; use crate::Result; -use crate::RoleEvent; use crate::StateMachineHandler; use crate::TypeConfig; use crate::alias::MOF; @@ -28,7 +28,7 @@ pub struct CommitHandlerDependencies { pub state_machine_handler: Arc>, pub raft_log: Arc>, pub membership: Arc>, - pub role_tx: mpsc::UnboundedSender, + pub internal_event_tx: mpsc::UnboundedSender, pub sm_apply_tx: mpsc::UnboundedSender>, pub shutdown_signal: watch::Receiver<()>, pub max_batch_size: usize, @@ -47,8 +47,8 @@ where new_commit_rx: Option>, membership: Arc>, - role_tx: mpsc::UnboundedSender, // Cloned from Raft - sm_apply_tx: mpsc::UnboundedSender>, // Send entries to SM Worker + internal_event_tx: mpsc::UnboundedSender, // Cloned from Raft + sm_apply_tx: mpsc::UnboundedSender>, // Send entries to SM Worker // Shutdown signal shutdown_signal: watch::Receiver<()>, @@ -130,7 +130,7 @@ where raft_log: deps.raft_log, membership: deps.membership, new_commit_rx: Some(new_commit_rx), - role_tx: deps.role_tx, + internal_event_tx: deps.internal_event_tx, sm_apply_tx: deps.sm_apply_tx, shutdown_signal: deps.shutdown_signal, max_batch_size: deps.max_batch_size, @@ -198,7 +198,7 @@ where self.send_to_sm_worker(&mut command_batch).await?; } - // Snapshot check moved to ApplyCompleted handler in Raft event loop. + // Snapshot check moved to ApplyCompleted handler in inbound event loop. // SM Worker applies entries asynchronously, so last_applied is stale here. Ok(()) @@ -254,7 +254,7 @@ where // 2.5. Notify leader to refresh cluster metadata cache // This must happen AFTER membership is applied - if let Err(e) = self.role_tx.send(RoleEvent::MembershipApplied) { + if let Err(e) = self.internal_event_tx.send(InternalEvent::MembershipApplied) { warn!("Failed to send MembershipApplied event: {:?}", e); } @@ -265,7 +265,7 @@ where self.my_id, entry.index ); // Signal step down - error is non-fatal as removal is already committed - if let Err(e) = self.role_tx.send(RoleEvent::StepDownSelfRemoved) { + if let Err(e) = self.internal_event_tx.send(InternalEvent::StepDownSelfRemoved) { error!( "[{}] Failed to send StepDownSelfRemoved event: {:?}", self.my_id, e diff --git a/d-engine-core/src/commit_handler/default_commit_handler_test.rs b/d-engine-core/src/commit_handler/default_commit_handler_test.rs index 077e0b61..d04386ec 100644 --- a/d-engine-core/src/commit_handler/default_commit_handler_test.rs +++ b/d-engine-core/src/commit_handler/default_commit_handler_test.rs @@ -14,13 +14,13 @@ use super::CommitHandler; use super::CommitHandlerDependencies; use super::DefaultCommitHandler; use crate::Error; +use crate::InternalEvent; use crate::MockMembership; use crate::MockRaftLog; use crate::MockStateMachineHandler; use crate::MockTypeConfig; use crate::NewCommitData; use crate::Result; -use crate::RoleEvent; const TEST_TERM: u64 = 1; @@ -68,8 +68,8 @@ pub struct TestHarness { mock_membership: Arc>, commit_tx: mpsc::UnboundedSender, commit_rx: Option>, - role_tx: mpsc::UnboundedSender, - role_rx: mpsc::UnboundedReceiver, + internal_event_tx: mpsc::UnboundedSender, + internal_event_rx: mpsc::UnboundedReceiver, shutdown_tx: watch::Sender<()>, shutdown_rx: Option>, handle: Option>, @@ -90,7 +90,7 @@ where { let (commit_tx, commit_rx) = mpsc::unbounded_channel(); let (shutdown_tx, shutdown_rx) = watch::channel(()); - let (role_tx, role_rx) = mpsc::unbounded_channel(); // Large capacity to prevent blocking in tests + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); // Large capacity to prevent blocking in tests // Mock state machine let mut mock_smh = MockStateMachineHandler::new(); @@ -134,8 +134,8 @@ where mock_membership: Arc::new(mock_membership), commit_rx: Some(commit_rx), commit_tx, - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, shutdown_tx, shutdown_rx: Some(shutdown_rx), handle: None, @@ -150,7 +150,7 @@ impl TestHarness { state_machine_handler: self.mock_smh.clone(), raft_log: self.mock_log.clone(), membership: self.mock_membership.clone(), - role_tx: self.role_tx.clone(), + internal_event_tx: self.internal_event_tx.clone(), sm_apply_tx, shutdown_signal: self.shutdown_rx.take().unwrap(), max_batch_size: 10, @@ -185,7 +185,7 @@ impl TestHarness { state_machine_handler: self.mock_smh.clone(), raft_log: self.mock_log.clone(), membership: self.mock_membership.clone(), - role_tx: self.role_tx.clone(), + internal_event_tx: self.internal_event_tx.clone(), sm_apply_tx, shutdown_signal: self.shutdown_rx.take().unwrap(), max_batch_size: 10, @@ -224,7 +224,7 @@ impl TestHarness { } async fn expect_snapshot_trigger(&mut self) -> bool { - match time::timeout(Duration::from_millis(50), self.role_rx.recv()).await { + match time::timeout(Duration::from_millis(50), self.internal_event_rx.recv()).await { Ok(Some(_)) => true, // Event received normally Ok(None) => false, // Channel closed Err(_) => false, // Timeout @@ -650,7 +650,7 @@ mod process_batch_test { // snapshot_condition: Option, // ) -> ( // DefaultCommitHandler, - // mpsc::Receiver, + // mpsc::Receiver, // watch::Sender<()>, // ) where // F: Fn() -> bool + 'static + Send + Sync, @@ -910,7 +910,7 @@ mod process_batch_test { /// Test 1: MembershipApplied event MUST be sent after successful config change /// /// Verifies that when apply_config_change() succeeds, the commit handler - /// sends RoleEvent::MembershipApplied to notify Leader to refresh cache. + /// sends InternalEvent::MembershipApplied to notify Leader to refresh cache. /// /// Related: Bug fix #209 - cluster metadata cache timing issue #[tokio::test] @@ -939,8 +939,10 @@ mod process_batch_test { assert!(result.is_ok(), "Config change should succeed"); // Verify MembershipApplied event was sent - match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { - Ok(Some(RoleEvent::MembershipApplied)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await + { + Ok(Some(InternalEvent::MembershipApplied)) => { // Success - event received } Ok(Some(other)) => panic!("Expected MembershipApplied, got {other:?}"), @@ -993,8 +995,10 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); // Verify event received - match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { - Ok(Some(RoleEvent::MembershipApplied)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await + { + Ok(Some(InternalEvent::MembershipApplied)) => { // Record event reception apply_order.lock().push("MembershipApplied_event"); } @@ -1042,7 +1046,9 @@ mod process_batch_test { assert!(result.is_err(), "Config change should fail"); // Verify NO event was sent - match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { + match tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await + { Err(_) => { // Timeout is expected - no event sent } @@ -1076,8 +1082,9 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); assert!( matches!( - tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await, - Ok(Some(RoleEvent::MembershipApplied)) + tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await, + Ok(Some(InternalEvent::MembershipApplied)) ), "AddNode should send MembershipApplied" ); @@ -1093,8 +1100,9 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); assert!( matches!( - tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await, - Ok(Some(RoleEvent::MembershipApplied)) + tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await, + Ok(Some(InternalEvent::MembershipApplied)) ), "RemoveNode should send MembershipApplied" ); @@ -1113,8 +1121,9 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); assert!( matches!( - tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await, - Ok(Some(RoleEvent::MembershipApplied)) + tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await, + Ok(Some(InternalEvent::MembershipApplied)) ), "PromoteLearner should send MembershipApplied" ); @@ -1146,8 +1155,8 @@ mod process_batch_test { // Replace the receiver with a dummy so the original event_tx sends will fail. // Swap old_rx out of the harness so harness is still usable, then drop old_rx. - let (_dummy_tx, dummy_rx) = mpsc::unbounded_channel::(); - let old_rx = std::mem::replace(&mut harness.role_rx, dummy_rx); + let (_dummy_tx, dummy_rx) = mpsc::unbounded_channel::(); + let old_rx = std::mem::replace(&mut harness.internal_event_rx, dummy_rx); drop(old_rx); // Now harness.event_tx's peer receiver is gone → sends fail. let result = harness.process_batch_handler().await; @@ -1181,8 +1190,8 @@ mod process_batch_test { ); // Replace receiver with a dummy so both sends fail. - let (_dummy_tx, dummy_rx) = mpsc::unbounded_channel::(); - let old_rx = std::mem::replace(&mut harness.role_rx, dummy_rx); + let (_dummy_tx, dummy_rx) = mpsc::unbounded_channel::(); + let old_rx = std::mem::replace(&mut harness.internal_event_rx, dummy_rx); drop(old_rx); let result = harness.process_batch_handler().await; @@ -1225,8 +1234,10 @@ mod process_batch_test { harness.process_batch_handler().await.unwrap(); // First event should be MembershipApplied - match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { - Ok(Some(RoleEvent::MembershipApplied)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await + { + Ok(Some(InternalEvent::MembershipApplied)) => { // Correct first event } Ok(Some(other)) => panic!("Expected MembershipApplied first, got {other:?}"), @@ -1234,8 +1245,10 @@ mod process_batch_test { } // Second event should be StepDownSelfRemoved - match tokio::time::timeout(Duration::from_millis(50), harness.role_rx.recv()).await { - Ok(Some(RoleEvent::StepDownSelfRemoved)) => { + match tokio::time::timeout(Duration::from_millis(50), harness.internal_event_rx.recv()) + .await + { + Ok(Some(InternalEvent::StepDownSelfRemoved)) => { // Correct second event } Ok(Some(other)) => panic!("Expected StepDownSelfRemoved second, got {other:?}"), diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index aad2cc64..ae514dac 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -316,7 +316,7 @@ impl ReplicationConfig { /// A single value intentionally covers all drain loops because they all operate /// within the same heartbeat period and share the same order-of-magnitude concurrency: /// - `raft.rs` cmd_rx drain (client propose ingestion) -/// - `raft.rs` role_rx drain (commit index event coalescing) +/// - `raft.rs` internal_event_rx drain (commit index event coalescing) /// - `DefaultCommitHandler` new_commit_rx drain (state machine apply batching) /// /// Also used as the initial Vec capacity for propose and linearizable-read buffers diff --git a/d-engine-core/src/event.rs b/d-engine-core/src/event.rs index 5cfb8253..ea2988d9 100644 --- a/d-engine-core/src/event.rs +++ b/d-engine-core/src/event.rs @@ -33,7 +33,7 @@ pub struct NewCommitData { } /// Client commands that require batching for performance -/// Separated from internal RaftEvent for drain-driven processing +/// Separated from internal InboundEvent for drain-driven processing #[derive(Debug)] pub enum ClientCmd { Propose( @@ -52,7 +52,7 @@ pub enum ClientCmd { #[derive(Debug)] #[allow(dead_code)] -pub enum RoleEvent { +pub enum InternalEvent { BecomeFollower(Option), // BecomeFollower(Option) BecomeCandidate, BecomeLeader, @@ -65,7 +65,7 @@ pub enum RoleEvent { /// No state transition - pure notification for watch channel LeaderDiscovered(u32, u64), // (leader_id, term) - ReprocessEvent(Box), //Replay the raft event when step down as another role + ReprocessEvent(Box), //Replay the inbound event when step down as another role /// Notify Raft loop that log entries up to `durable_index` are crash-safe (fsync complete). /// Sent by batch_processor after flush completes. @@ -97,7 +97,7 @@ pub enum RoleEvent { term: u64, }, - /// State machine apply completed — processed at P2 (unbounded role_tx) to avoid + /// State machine apply completed — processed at P2 (unbounded internal_event_tx) to avoid /// priority inversion: AppendEntries RPCs at P4 (bounded event_tx) must not starve /// internal commit-driven events. ApplyCompleted { @@ -105,7 +105,7 @@ pub enum RoleEvent { results: Vec, }, - /// Trigger state machine snapshot creation — routed via P2 (unbounded role_tx) to + /// Trigger state machine snapshot creation — routed via P2 (unbounded internal_event_tx) to /// prevent deadlock when P4 event_tx is saturated by inbound RPCs. CreateSnapshotEvent, @@ -122,7 +122,7 @@ pub enum RoleEvent { MembershipApplied, /// Check pending learners for promotion eligibility after a membership change or heartbeat. - /// Re-queued via role_tx until all pending promotions are resolved. + /// Re-queued via internal_event_tx until all pending promotions are resolved. PromoteReadyLearners, /// Log entries up to `LogId` have been purged from storage after snapshot creation. @@ -143,7 +143,7 @@ pub enum RoleEvent { ZombieDetected(u32), /// Fatal error from SM worker — node must shutdown. - /// Sent via role_tx (P2) so it is not blocked behind external RPCs on event_tx (P4). + /// Sent via internal_event_tx (P2) so it is not blocked behind external RPCs on event_tx (P4). FatalError { source: String, error: String, @@ -151,7 +151,7 @@ pub enum RoleEvent { } #[derive(Debug)] -pub enum RaftEvent { +pub enum InboundEvent { ReceiveVoteRequest( VoteRequest, MaybeCloneOneshotSender>, @@ -248,17 +248,17 @@ pub enum TestEvent { } #[cfg(test)] -pub(crate) fn raft_event_to_test_event(event: &RaftEvent) -> TestEvent { +pub(crate) fn inbound_event_to_test_event(event: &InboundEvent) -> TestEvent { match event { - RaftEvent::ReceiveVoteRequest(req, _) => TestEvent::ReceiveVoteRequest(*req), - RaftEvent::ClusterConf(req, _) => TestEvent::ClusterConf(*req), - RaftEvent::ClusterConfUpdate(req, _) => TestEvent::ClusterConfUpdate(req.clone()), - RaftEvent::AppendEntries(req, _) => TestEvent::AppendEntries(req.clone()), - RaftEvent::InstallSnapshotChunk(_, _) => TestEvent::InstallSnapshotChunk, - RaftEvent::StreamSnapshot(_, _, _) => TestEvent::StreamSnapshot, - RaftEvent::JoinCluster(req, _) => TestEvent::JoinCluster(req.clone()), - RaftEvent::DiscoverLeader(req, _) => TestEvent::DiscoverLeader(req.clone()), - RaftEvent::FatalError { source, error } => TestEvent::FatalError { + InboundEvent::ReceiveVoteRequest(req, _) => TestEvent::ReceiveVoteRequest(*req), + InboundEvent::ClusterConf(req, _) => TestEvent::ClusterConf(*req), + InboundEvent::ClusterConfUpdate(req, _) => TestEvent::ClusterConfUpdate(req.clone()), + InboundEvent::AppendEntries(req, _) => TestEvent::AppendEntries(req.clone()), + InboundEvent::InstallSnapshotChunk(_, _) => TestEvent::InstallSnapshotChunk, + InboundEvent::StreamSnapshot(_, _, _) => TestEvent::StreamSnapshot, + InboundEvent::JoinCluster(req, _) => TestEvent::JoinCluster(req.clone()), + InboundEvent::DiscoverLeader(req, _) => TestEvent::DiscoverLeader(req.clone()), + InboundEvent::FatalError { source, error } => TestEvent::FatalError { source: source.clone(), error: error.clone(), }, diff --git a/d-engine-core/src/membership.rs b/d-engine-core/src/membership.rs index 672237d8..85ff000c 100644 --- a/d-engine-core/src/membership.rs +++ b/d-engine-core/src/membership.rs @@ -43,7 +43,7 @@ where /// All non-self nodes (including Syncing and Active) /// Note: - /// Joining node has not start its Raft event processing engine yet. + /// Joining node has not start its inbound event processing engine yet. async fn replication_peers(&self) -> Vec; /// All non-self nodes in Active state diff --git a/d-engine-core/src/network/mod.rs b/d-engine-core/src/network/mod.rs index 7c723993..82877f55 100644 --- a/d-engine-core/src/network/mod.rs +++ b/d-engine-core/src/network/mod.rs @@ -282,7 +282,7 @@ where /// Used when a peer's `next_index` falls below the leader's purge boundary and /// AppendEntries would carry a stale `prev_log_term = 0`, causing a perpetual /// conflict loop. The worker calls this and awaits completion before emitting - /// `RoleEvent::SnapshotPushCompleted`. + /// `InternalEvent::SnapshotPushCompleted`. /// /// # Parameters /// - `peer_id`: Target follower node ID @@ -333,7 +333,7 @@ where /// /// When the stream breaks (network error, peer restart), the receiver yields /// an `Err(tonic::Status)` and the caller should emit - /// `RoleEvent::PeerStreamError { peer_id }` so the Raft loop can reset + /// `InternalEvent::PeerStreamError { peer_id }` so the Raft loop can reset /// `next_index` and schedule reconnection. /// /// # Errors diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index 2e560ff2..b9636917 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -10,15 +10,15 @@ use tracing::info; use tracing::trace; use tracing::warn; +use super::InboundEvent; +use super::InternalEvent; use super::NewCommitData; use super::RaftContext; use super::RaftCoreHandlers; -use super::RaftEvent; use super::RaftRole; use super::RaftStorageHandles; -use super::RoleEvent; #[cfg(test)] -use super::raft_event_to_test_event; +use super::inbound_event_to_test_event; use crate::Membership; use crate::NetworkError; use crate::RaftLog; @@ -39,18 +39,18 @@ where pub ctx: RaftContext, // Network & Storage events - event_tx: mpsc::Sender, - event_rx: mpsc::Receiver, - buffered_raft_event: VecDeque, + event_tx: mpsc::Sender, + event_rx: mpsc::Receiver, + buffered_inbound_event: VecDeque, // Client commands (drain-driven) cmd_tx: mpsc::Sender, cmd_rx: mpsc::Receiver, // Timer - role_tx: mpsc::UnboundedSender, - role_rx: mpsc::UnboundedReceiver, - buffered_role_event: VecDeque, + internal_event_tx: mpsc::UnboundedSender, + internal_event_rx: mpsc::UnboundedReceiver, + buffered_internal_event: VecDeque, // For business logic to apply logs into state machine new_commit_listener: Vec>, @@ -67,14 +67,14 @@ where test_role_transition_listener: Vec>, #[cfg(test)] - test_raft_event_listener: Vec>, + test_inbound_event_listener: Vec>, } pub struct SignalParams { - pub(crate) role_tx: mpsc::UnboundedSender, - pub(crate) role_rx: mpsc::UnboundedReceiver, - pub(crate) event_tx: mpsc::Sender, - pub(crate) event_rx: mpsc::Receiver, + pub(crate) internal_event_tx: mpsc::UnboundedSender, + pub(crate) internal_event_rx: mpsc::UnboundedReceiver, + pub(crate) event_tx: mpsc::Sender, + pub(crate) event_rx: mpsc::Receiver, pub(crate) cmd_tx: mpsc::Sender, pub(crate) cmd_rx: mpsc::Receiver, pub(crate) shutdown_signal: watch::Receiver<()>, @@ -86,17 +86,17 @@ impl SignalParams { /// This is the only way to construct SignalParams from outside d-engine-core, /// ensuring controlled initialization of the internal communication channels. pub fn new( - role_tx: mpsc::UnboundedSender, - role_rx: mpsc::UnboundedReceiver, - event_tx: mpsc::Sender, - event_rx: mpsc::Receiver, + internal_event_tx: mpsc::UnboundedSender, + internal_event_rx: mpsc::UnboundedReceiver, + event_tx: mpsc::Sender, + event_rx: mpsc::Receiver, cmd_tx: mpsc::Sender, cmd_rx: mpsc::Receiver, shutdown_signal: watch::Receiver<()>, ) -> Self { Self { - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, event_tx, event_rx, cmd_tx, @@ -137,14 +137,14 @@ where event_tx: signal_params.event_tx, event_rx: signal_params.event_rx, - buffered_raft_event: VecDeque::new(), + buffered_inbound_event: VecDeque::new(), cmd_tx: signal_params.cmd_tx, cmd_rx: signal_params.cmd_rx, - role_tx: signal_params.role_tx, - role_rx: signal_params.role_rx, - buffered_role_event: VecDeque::new(), + internal_event_tx: signal_params.internal_event_tx, + internal_event_rx: signal_params.internal_event_rx, + buffered_internal_event: VecDeque::new(), new_commit_listener: Vec::new(), @@ -156,7 +156,7 @@ where test_role_transition_listener: Vec::new(), #[cfg(test)] - test_raft_event_listener: Vec::new(), + test_inbound_event_listener: Vec::new(), } } @@ -281,21 +281,21 @@ where // P1: Tick: start Heartbeat(replication) or start Election _ = tick => { trace!("receive tick"); - let role_tx = &self.role_tx; + let internal_event_tx = &self.internal_event_tx; let event_tx = &self.event_tx; - if let Err(e) = self.role.tick(role_tx, event_tx, &self.ctx).await { + if let Err(e) = self.role.tick(internal_event_tx, event_tx, &self.ctx).await { error!("tick failed: {:?}", e); } else { trace!("tick success"); } } - // P2: Role events — handle first, drain rest after select - Some(role_event) = self.role_rx.recv() => { - debug!(%self.node_id, ?role_event, "receive role event"); - self.buffered_role_event.push_back(role_event); - self.drain_role_events().await?; + // P2: internal events — handle first, drain rest after select + Some(internal_event) = self.internal_event_rx.recv() => { + debug!(%self.node_id, ?internal_event, "receive internal event"); + self.buffered_internal_event.push_back(internal_event); + self.drain_internal_events().await?; } // P3: Client commands — push first, drain rest after select @@ -306,28 +306,28 @@ where } // P4: Other events — handle first, drain rest after select - Some(raft_event) = self.event_rx.recv() => { - trace!(%self.node_id, ?raft_event, "receive raft event"); - self.buffered_raft_event.push_back(raft_event); - self.drain_raft_events().await?; + Some(inbound_event) = self.event_rx.recv() => { + trace!(%self.node_id, ?inbound_event, "receive inbound event"); + self.buffered_inbound_event.push_back(inbound_event); + self.drain_inbound_events().await?; } } - self.process_role_events().await?; + self.process_internal_events().await?; self.process_client_cmds().await?; - self.process_raft_events().await?; + self.process_inbound_events().await?; } } - /// Drain all pending raft events (up to max_batch_size). - async fn drain_raft_events(&mut self) -> Result<()> { + /// Drain all pending inbound events (up to max_batch_size). + async fn drain_inbound_events(&mut self) -> Result<()> { let max = self.ctx.node_config.raft.batching.max_batch_size; let mut count = 0; while count < max { match self.event_rx.try_recv() { - Ok(raft_event) => { - self.buffered_raft_event.push_back(raft_event); + Ok(inbound_event) => { + self.buffered_inbound_event.push_back(inbound_event); count += 1; } Err(_) => break, @@ -336,14 +336,14 @@ where Ok(()) } - /// Drain all pending role events (up to max_batch_size). - async fn drain_role_events(&mut self) -> Result<()> { + /// Drain all pending internal events (up to max_batch_size). + async fn drain_internal_events(&mut self) -> Result<()> { let max = self.ctx.node_config.raft.batching.max_batch_size; let mut count = 0; while count < max { - match self.role_rx.try_recv() { - Ok(role_event) => { - self.buffered_role_event.push_back(role_event); + match self.internal_event_rx.try_recv() { + Ok(internal_event) => { + self.buffered_internal_event.push_back(internal_event); count += 1; } Err(_) => break, @@ -371,48 +371,50 @@ where Ok(()) } - async fn process_role_events(&mut self) -> Result<()> { - while let Some(event) = self.buffered_role_event.pop_front() { - if let Err(e) = self.handle_role_event(event).await { + async fn process_internal_events(&mut self) -> Result<()> { + while let Some(event) = self.buffered_internal_event.pop_front() { + if let Err(e) = self.handle_internal_event(event).await { if e.is_fatal() { - error!(%self.node_id, ?e, "Fatal error in process_role_events, shutting down"); + error!(%self.node_id, ?e, "Fatal error in process_internal_events, shutting down"); return Err(e); } - warn!(%self.node_id, ?e, "Non-fatal error in process_role_events, continuing"); + warn!(%self.node_id, ?e, "Non-fatal error in process_internal_events, continuing"); } } Ok(()) } - async fn process_raft_events(&mut self) -> Result<()> { - while !self.buffered_raft_event.is_empty() { + async fn process_inbound_events(&mut self) -> Result<()> { + while !self.buffered_inbound_event.is_empty() { // Avoid none AE event pop and push into queue if matches!( - self.buffered_raft_event.front(), - Some(RaftEvent::AppendEntries(..)) + self.buffered_inbound_event.front(), + Some(InboundEvent::AppendEntries(..)) ) { self.merge_append_entries(); } - let Some(event) = self.buffered_raft_event.pop_front() else { + let Some(event) = self.buffered_inbound_event.pop_front() else { break; }; #[cfg(test)] - let test_event = raft_event_to_test_event(&event); + let test_event = inbound_event_to_test_event(&event); - if let Err(e) = - self.role.handle_raft_event(event, &self.ctx, self.role_tx.clone()).await + if let Err(e) = self + .role + .handle_inbound_event(event, &self.ctx, self.internal_event_tx.clone()) + .await { if e.is_fatal() { - error!(%self.node_id, ?e, "Fatal error in drain_raft_events, shutting down"); + error!(%self.node_id, ?e, "Fatal error in drain_inbound_events, shutting down"); return Err(e); } - warn!(%self.node_id, ?e, "Non-fatal error in drain_raft_events, continuing"); + warn!(%self.node_id, ?e, "Non-fatal error in drain_inbound_events, continuing"); } #[cfg(test)] - self.notify_raft_event(test_event); + self.notify_inbound_event(test_event); } Ok(()) } @@ -420,17 +422,17 @@ where async fn process_client_cmds(&mut self) -> Result<()> { // Always flush: the P3 select arm may have pushed a command before this drain ran. // flush_cmd_buffers checks is_empty() internally and is a no-op when nothing is buffered. - self.role.flush_cmd_buffers(&self.ctx, &self.role_tx).await + self.role.flush_cmd_buffers(&self.ctx, &self.internal_event_tx).await } - /// Drain all pending raft events (up to max_batch_size). + /// Drain all pending inbound events (up to max_batch_size). fn merge_append_entries(&mut self) { - let Some(event) = self.buffered_raft_event.pop_front() else { + let Some(event) = self.buffered_inbound_event.pop_front() else { return; }; - let RaftEvent::AppendEntries(mut first_req, first_senders) = event else { - self.buffered_raft_event.push_front(event); + let InboundEvent::AppendEntries(mut first_req, first_senders) = event else { + self.buffered_inbound_event.push_front(event); return; }; @@ -441,13 +443,14 @@ where let term = first_req.term; let mut merged_senders = first_senders; - while let Some(event) = self.buffered_raft_event.pop_front() { + while let Some(event) = self.buffered_inbound_event.pop_front() { match event { - RaftEvent::AppendEntries(mut req, mut senders) + InboundEvent::AppendEntries(mut req, mut senders) if next_prev == req.prev_log_index && term == req.term => { if merged_entries.len() + req.entries.len() > max { - self.buffered_raft_event.push_front(RaftEvent::AppendEntries(req, senders)); + self.buffered_inbound_event + .push_front(InboundEvent::AppendEntries(req, senders)); break; } @@ -461,26 +464,26 @@ where merged_senders.append(&mut senders); } _ => { - self.buffered_raft_event.push_front(event); + self.buffered_inbound_event.push_front(event); break; } } } first_req.entries = merged_entries; - self.buffered_raft_event - .push_front(RaftEvent::AppendEntries(first_req, merged_senders)); + self.buffered_inbound_event + .push_front(InboundEvent::AppendEntries(first_req, merged_senders)); } - /// `handle_role_event` will be responsbile to process role trasnsition and + /// `handle_internal_event` will be responsbile to process role trasnsition and /// role state events. - pub async fn handle_role_event( + pub async fn handle_internal_event( &mut self, - role_event: RoleEvent, + internal_event: InternalEvent, ) -> Result<()> { - // All inbound and outbound raft event + // All inbound and outbound inbound event - match role_event { - RoleEvent::BecomeFollower(leader_id_option) => { + match internal_event { + InternalEvent::BecomeFollower(leader_id_option) => { // Drain read buffer when stepping down from Leader; skip otherwise. let _ = self.role.drain_read_buffer(); @@ -499,7 +502,7 @@ where //TODO: update membership } - RoleEvent::BecomeCandidate => { + InternalEvent::BecomeCandidate => { // Drain read buffer when stepping down from Leader; skip otherwise. let _ = self.role.drain_read_buffer(); @@ -513,7 +516,7 @@ where #[cfg(test)] self.notify_role_transition(); } - RoleEvent::BecomeLeader => { + InternalEvent::BecomeLeader => { debug!("BecomeLeader"); self.role = self.role.become_leader()?; @@ -536,19 +539,23 @@ where self.role.state_mut().init_cluster_metadata(&self.ctx.membership()).await?; // Fire-and-forget noop to confirm quorum. - // Completion is delivered asynchronously via RoleEvent::NoopCommitted. - if let Err(e) = self.role.initiate_noop_commit(&self.ctx, &self.role_tx).await { + // Completion is delivered asynchronously via InternalEvent::NoopCommitted. + if let Err(e) = + self.role.initiate_noop_commit(&self.ctx, &self.internal_event_tx).await + { warn!(?e, "initiate_noop_commit failed — stepping down"); - self.role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { - error!("Failed to send: {:?}", e); - NetworkError::SingalSendFailed(format!("{:?}", e)) - })?; + self.internal_event_tx.send(InternalEvent::BecomeFollower(None)).map_err( + |e| { + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) + }, + )?; } #[cfg(test)] self.notify_role_transition(); } - RoleEvent::BecomeLearner => { + InternalEvent::BecomeLearner => { // Drain read buffer when stepping down from Leader; skip otherwise. let _ = self.role.drain_read_buffer(); @@ -562,15 +569,15 @@ where #[cfg(test)] self.notify_role_transition(); } - RoleEvent::NotifyNewCommitIndex(mut new_commit_data) => { + InternalEvent::NotifyNewCommitIndex(mut new_commit_data) => { // Drain all pending NotifyNewCommitIndex events (max_batch_size limit) // This batches multiple committed entries into a single notification let max_batch = self.ctx.node_config.raft.batching.max_batch_size; let mut count = 1; while count < max_batch { - match self.role_rx.try_recv() { - Ok(RoleEvent::NotifyNewCommitIndex(next)) => { + match self.internal_event_rx.try_recv() { + Ok(InternalEvent::NotifyNewCommitIndex(next)) => { // Only keep the largest commit_index if next.new_commit_index > new_commit_data.new_commit_index { new_commit_data = next; @@ -578,8 +585,8 @@ where count += 1; } Ok(other) => { - self.role_tx.send(other).map_err(|e| { - error!("Failed to resend role event: {:?}", e); + self.internal_event_tx.send(other).map_err(|e| { + error!("Failed to resend internal event: {:?}", e); crate::Error::Fatal(e.to_string()) })?; break; @@ -595,51 +602,53 @@ where self.notify_new_commit(new_commit_data); } - RoleEvent::LeaderDiscovered(leader_id, term) => { + InternalEvent::LeaderDiscovered(leader_id, term) => { debug!("LeaderDiscovered: leader_id={}, term={}", leader_id, term); // Notify leader change listeners - no state transition // Note: mpsc channels do not deduplicate; consumers handle dedup if needed self.notify_leader_change(Some(leader_id), term); } - RoleEvent::ReprocessEvent(raft_event) => { - info!("Replay the RaftEvent: {:?}", &raft_event); - self.buffered_raft_event.push_front(*raft_event); + InternalEvent::ReprocessEvent(inbound_event) => { + info!("Replay the InboundEvent: {:?}", &inbound_event); + self.buffered_inbound_event.push_front(*inbound_event); } - RoleEvent::LogFlushed { durable_index } => { + InternalEvent::LogFlushed { durable_index } => { debug!("LogFlushed: durable_index={}", durable_index); - self.role.handle_log_flushed(durable_index, &self.ctx, &self.role_tx).await; + self.role + .handle_log_flushed(durable_index, &self.ctx, &self.internal_event_tx) + .await; } - RoleEvent::AppendResult { + InternalEvent::AppendResult { follower_id, result, } => { debug!("AppendResult: follower_id={}", follower_id); if let Err(e) = self .role - .handle_append_result(follower_id, result, &self.ctx, &self.role_tx) + .handle_append_result(follower_id, result, &self.ctx, &self.internal_event_tx) .await { error!("handle_append_result failed: {:?}", e); } } - RoleEvent::NoopCommitted { term } => { + InternalEvent::NoopCommitted { term } => { debug!("NoopCommitted: term={}", term); // on_noop_committed already called directly in drain_commit_actions. // Only notify leader change listeners here (requires Raft access). self.notify_leader_change(Some(self.node_id), term); } - RoleEvent::FatalError { source, error } => { + InternalEvent::FatalError { source, error } => { error!(%self.node_id, %source, %error, "Fatal error from SM worker — shutting down"); return Err(crate::Error::Fatal(format!("{source}: {error}"))); } - RoleEvent::ApplyCompleted { + InternalEvent::ApplyCompleted { last_index, results, } => { - // Routed via role_tx (P2) to avoid priority inversion against AppendEntries at P4. + // Routed via internal_event_tx (P2) to avoid priority inversion against AppendEntries at P4. if let Err(e) = self .role - .handle_apply_completed(last_index, results, &self.ctx, &self.role_tx) + .handle_apply_completed(last_index, results, &self.ctx, &self.internal_event_tx) .await { if e.is_fatal() { @@ -649,19 +658,21 @@ where warn!(%self.node_id, ?e, "Non-fatal error in ApplyCompleted handler"); } } - RoleEvent::PeerStreamError { peer_id } => { + InternalEvent::PeerStreamError { peer_id } => { debug!(%peer_id, "PeerStreamError: bidi stream disconnected, resetting next_index"); self.role.handle_peer_stream_error(peer_id); } - RoleEvent::ZombieDetected(node_id) => { + InternalEvent::ZombieDetected(node_id) => { debug!(%node_id, "ZombieDetected: forwarding to leader for BatchRemove"); - if let Err(e) = - self.role.handle_zombie_detected(node_id, &self.role_tx, &self.ctx).await + if let Err(e) = self + .role + .handle_zombie_detected(node_id, &self.internal_event_tx, &self.ctx) + .await { error!(%node_id, ?e, "handle_zombie_detected failed"); } } - RoleEvent::SnapshotPushCompleted { peer_id, success } => { + InternalEvent::SnapshotPushCompleted { peer_id, success } => { debug!(%peer_id, %success, "SnapshotPushCompleted"); if success { // Reset next_index to last_entry_id + 1 so the peer is no longer below @@ -678,17 +689,21 @@ where let policy = &self.ctx.node_config.retry.install_snapshot; self.role.handle_snapshot_push_completed(peer_id, success, policy, self.node_id); } - RoleEvent::CreateSnapshotEvent => { - if let Err(e) = self.role.handle_create_snapshot(&self.ctx, &self.role_tx).await { + InternalEvent::CreateSnapshotEvent => { + if let Err(e) = + self.role.handle_create_snapshot(&self.ctx, &self.internal_event_tx).await + { if e.is_fatal() { return Err(e); } error!(%self.node_id, ?e, "handle_create_snapshot failed"); } } - RoleEvent::SnapshotCreated(result) => { - if let Err(e) = - self.role.handle_snapshot_created(result, &self.ctx, &self.role_tx).await + InternalEvent::SnapshotCreated(result) => { + if let Err(e) = self + .role + .handle_snapshot_created(result, &self.ctx, &self.internal_event_tx) + .await { if e.is_fatal() { return Err(e); @@ -696,16 +711,17 @@ where error!(%self.node_id, ?e, "handle_snapshot_created failed"); } } - RoleEvent::StepDownSelfRemoved => { - if let Err(e) = self.role.handle_self_removed(&self.role_tx) { + InternalEvent::StepDownSelfRemoved => { + if let Err(e) = self.role.handle_self_removed(&self.internal_event_tx) { if e.is_fatal() { return Err(e); } error!(%self.node_id, ?e, "handle_self_removed failed"); } } - RoleEvent::MembershipApplied => { - if let Err(e) = self.role.handle_membership_applied(&self.ctx, &self.role_tx).await + InternalEvent::MembershipApplied => { + if let Err(e) = + self.role.handle_membership_applied(&self.ctx, &self.internal_event_tx).await { if e.is_fatal() { return Err(e); @@ -713,9 +729,11 @@ where error!(%self.node_id, ?e, "handle_membership_applied failed"); } } - RoleEvent::PromoteReadyLearners => { - if let Err(e) = - self.role.handle_promote_ready_learners(&self.ctx, &self.role_tx).await + InternalEvent::PromoteReadyLearners => { + if let Err(e) = self + .role + .handle_promote_ready_learners(&self.ctx, &self.internal_event_tx) + .await { if e.is_fatal() { return Err(e); @@ -723,7 +741,7 @@ where error!(%self.node_id, ?e, "handle_promote_ready_learners failed"); } } - RoleEvent::LogPurgeCompleted(log_id) => { + InternalEvent::LogPurgeCompleted(log_id) => { if let Err(e) = self.role.handle_log_purge_completed(log_id) { if e.is_fatal() { return Err(e); @@ -773,21 +791,21 @@ where } #[cfg(test)] - pub fn register_raft_event_listener( + pub fn register_inbound_event_listener( &mut self, tx: mpsc::UnboundedSender, ) { - self.test_raft_event_listener.push(tx); + self.test_inbound_event_listener.push(tx); } #[cfg(test)] - pub fn notify_raft_event( + pub fn notify_inbound_event( &self, event: super::TestEvent, ) { - debug!("unit test:: notify new raft event: {:?}", &event); + debug!("unit test:: notify new inbound event: {:?}", &event); - for tx in &self.test_raft_event_listener { + for tx in &self.test_inbound_event_listener { assert!(tx.send(event.clone()).is_ok(), "should succeed"); } } @@ -802,15 +820,15 @@ where /// Returns a cloned event sender for external use. /// - /// This provides controlled access to send validated RaftEvents to the Raft core. + /// This provides controlled access to send validated InboundEvents to the Raft core. /// Events sent through this sender are still processed through the normal validation /// pipeline in the main event loop. /// /// # Security Note /// While this provides access to the event channel, all events are still validated - /// by the Raft state machine before being applied. The event handler in `handle_raft_event` + /// by the Raft state machine before being applied. The event handler in `handle_inbound_event` /// performs necessary checks based on current term, role, and state. - pub fn event_sender(&self) -> mpsc::Sender { + pub fn event_sender(&self) -> mpsc::Sender { self.event_tx.clone() } @@ -826,14 +844,14 @@ where self.role.state().current_term() } - /// Returns a cloned role event sender for internal use. + /// Returns a cloned internal event sender for internal use. /// /// # Warning /// This is primarily for internal components that need to trigger role transitions. /// External callers should not use this unless they understand the Raft protocol deeply. #[doc(hidden)] - pub fn role_event_sender(&self) -> mpsc::UnboundedSender { - self.role_tx.clone() + pub fn internal_event_sender(&self) -> mpsc::UnboundedSender { + self.internal_event_tx.clone() } } @@ -866,8 +884,8 @@ mod leader_discovered_tests; #[path = "raft_test/merge_append_entries_tests.rs"] mod merge_append_entries_tests; #[cfg(test)] -#[path = "raft_test/process_raft_events_tests.rs"] -mod process_raft_events_tests; +#[path = "raft_test/process_inbound_events_tests.rs"] +mod process_inbound_events_tests; #[cfg(test)] #[path = "raft_test/raft_comprehensive_tests.rs"] mod raft_comprehensive_tests; diff --git a/d-engine-core/src/raft_role/candidate_state.rs b/d-engine-core/src/raft_role/candidate_state.rs index 8d587cc9..30df506a 100644 --- a/d-engine-core/src/raft_role/candidate_state.rs +++ b/d-engine-core/src/raft_role/candidate_state.rs @@ -28,14 +28,14 @@ use crate::ElectionCore; use crate::ElectionError; use crate::ElectionTimer; use crate::Error; +use crate::InboundEvent; +use crate::InternalEvent; use crate::Membership; use crate::NetworkError; use crate::RaftContext; -use crate::RaftEvent; use crate::RaftLog; use crate::RaftNodeConfig; use crate::ReplicationCore; -use crate::RoleEvent; use crate::StateTransitionError; use crate::TypeConfig; @@ -154,8 +154,8 @@ impl RaftRoleState for CandidateState { /// Election Timeout: as candidate, it should send vote requests now async fn tick( &mut self, - role_tx: &mpsc::UnboundedSender, - _raft_event_tx: &mpsc::Sender, + internal_event_tx: &mpsc::UnboundedSender, + _inbound_event_tx: &mpsc::Sender, ctx: &RaftContext, ) -> Result<()> { if Instant::now() < self.timer.next_deadline() { @@ -186,7 +186,7 @@ impl RaftRoleState for CandidateState { { Ok(_) => { debug!("BecomeLeader"); - if let Err(e) = role_tx.send(RoleEvent::BecomeLeader) { + if let Err(e) = internal_event_tx.send(InternalEvent::BecomeLeader) { error!( "self.my_role_change_event_sender.send(RaftRole::Leader) failed: {:?}", e @@ -198,7 +198,7 @@ impl RaftRoleState for CandidateState { )))) => { // Immediately update the Term and become a Follower. self.update_current_term(higher_term); - self.send_become_follower_event(role_tx)?; + self.send_become_follower_event(internal_event_tx)?; } Err(e) => { warn!("candidate broadcast_vote_requests with error: {:?}", e); @@ -207,17 +207,17 @@ impl RaftRoleState for CandidateState { Ok(()) } - async fn handle_raft_event( + async fn handle_inbound_event( &mut self, - raft_event: RaftEvent, + inbound_event: InboundEvent, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, ) -> Result<()> { let my_term = self.current_term(); - match raft_event { - RaftEvent::ReceiveVoteRequest(vote_request, sender) => { + match inbound_event { + InboundEvent::ReceiveVoteRequest(vote_request, sender) => { debug!( - "handle_raft_event::RaftEvent::ReceiveVoteRequest: {:?}", + "handle_inbound_event::InboundEvent::ReceiveVoteRequest: {:?}", &vote_request ); @@ -235,14 +235,14 @@ impl RaftRoleState for CandidateState { ) { self.update_current_term(vote_request.term); // Step down as Follower - self.send_become_follower_event(&role_tx)?; + self.send_become_follower_event(&internal_event_tx)?; info!( "Candidate will not process ReceiveVoteRequest, it should let Follower do it." ); - self.send_replay_raft_event( - &role_tx, - RaftEvent::ReceiveVoteRequest(vote_request, sender), + self.send_replay_inbound_event( + &internal_event_tx, + InboundEvent::ReceiveVoteRequest(vote_request, sender), )?; } else { let response = VoteResponse { @@ -258,7 +258,7 @@ impl RaftRoleState for CandidateState { } } - RaftEvent::ClusterConf(_metadata_request, sender) => { + InboundEvent::ClusterConf(_metadata_request, sender) => { let cluster_conf = ctx .membership() .retrieve_cluster_membership_config(self.shared_state().current_leader()) @@ -274,7 +274,7 @@ impl RaftRoleState for CandidateState { } } - RaftEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { + InboundEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { let current_conf_version = ctx.membership().get_cluster_conf_version().await; let current_leader_id = self.shared_state().current_leader(); @@ -319,9 +319,9 @@ impl RaftRoleState for CandidateState { } } - RaftEvent::AppendEntries(append_entries_request, senders) => { + InboundEvent::AppendEntries(append_entries_request, senders) => { debug!( - "handle_raft_event::RaftEvent::AppendEntries: {:?}", + "handle_inbound_event::InboundEvent::AppendEntries: {:?}", &append_entries_request ); @@ -342,14 +342,14 @@ impl RaftRoleState for CandidateState { self.update_current_term(append_entries_request.term); } // Step down as Follower - self.send_become_follower_event(&role_tx)?; + self.send_become_follower_event(&internal_event_tx)?; info!( "Candidate will not process AppendEntries request, it should let Follower do it." ); - self.send_replay_raft_event( - &role_tx, - RaftEvent::AppendEntries(append_entries_request, senders), + self.send_replay_inbound_event( + &internal_event_tx, + InboundEvent::AppendEntries(append_entries_request, senders), )?; } else { // request.term < my_term: stale leader, reject. @@ -371,7 +371,7 @@ impl RaftRoleState for CandidateState { } } - RaftEvent::InstallSnapshotChunk(_streaming, sender) => { + InboundEvent::InstallSnapshotChunk(_streaming, sender) => { sender .send(Err(Status::permission_denied("Not Follower or Learner."))) .map_err(|e| { @@ -383,14 +383,14 @@ impl RaftRoleState for CandidateState { current_role: "Candidate", required_role: "Follower or Learner", context: format!( - "Candidate node {} receives RaftEvent::InstallSnapshotChunk", + "Candidate node {} receives InboundEvent::InstallSnapshotChunk", ctx.node_id ), } .into()); } - RaftEvent::JoinCluster(_join_request, sender) => { + InboundEvent::JoinCluster(_join_request, sender) => { sender .send(Err(Status::permission_denied( "Candidate should not receive JoinCluster event.", @@ -404,15 +404,15 @@ impl RaftRoleState for CandidateState { current_role: "Candidate", required_role: "Leader", context: format!( - "Candidate node {} receives RaftEvent::JoinCluster", + "Candidate node {} receives InboundEvent::JoinCluster", ctx.node_id ), } .into()); } - RaftEvent::DiscoverLeader(request, sender) => { - debug!(?request, "Candidate::RaftEvent::DiscoverLeader"); + InboundEvent::DiscoverLeader(request, sender) => { + debug!(?request, "Candidate::InboundEvent::DiscoverLeader"); sender .send(Err(Status::permission_denied( "Candidate should not response DiscoverLeader event.", @@ -425,8 +425,8 @@ impl RaftRoleState for CandidateState { return Ok(()); } - RaftEvent::StreamSnapshot(_ack_rx, _chunk_tx, startup_tx) => { - debug!("Candidate::RaftEvent::StreamSnapshot"); + InboundEvent::StreamSnapshot(_ack_rx, _chunk_tx, startup_tx) => { + debug!("Candidate::InboundEvent::StreamSnapshot"); warn!("Candidate should not receive StreamSnapshot event."); if let Err(e) = startup_tx.send(Err(Status::failed_precondition("Not the leader"))) { @@ -438,7 +438,7 @@ impl RaftRoleState for CandidateState { return Ok(()); } - RaftEvent::FatalError { source, error } => { + InboundEvent::FatalError { source, error } => { error!("[Candidate] Fatal error from {}: {}", source, error); return Err(crate::Error::Fatal(format!( "Fatal error from {source}: {error}" @@ -493,24 +493,29 @@ impl CandidateState { pub fn send_become_follower_event( &self, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { + internal_event_tx.send(InternalEvent::BecomeFollower(None)).map_err(|e| { error!("Failed to send: {:?}", e); NetworkError::SingalSendFailed(format!("{:?}", e)).into() }) } - pub fn send_replay_raft_event( + pub fn send_replay_inbound_event( &self, - role_tx: &mpsc::UnboundedSender, - raft_event: RaftEvent, + internal_event_tx: &mpsc::UnboundedSender, + inbound_event: InboundEvent, ) -> Result<()> { - debug!("send_replay_raft_event, raft_event:{:?}", &raft_event); - role_tx.send(RoleEvent::ReprocessEvent(Box::new(raft_event))).map_err(|e| { - error!("Failed to send: {:?}", e); - NetworkError::SingalSendFailed(format!("{:?}", e)).into() - }) + debug!( + "send_replay_inbound_event, inbound_event:{:?}", + &inbound_event + ); + internal_event_tx + .send(InternalEvent::ReprocessEvent(Box::new(inbound_event))) + .map_err(|e| { + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)).into() + }) } #[cfg(test)] diff --git a/d-engine-core/src/raft_role/candidate_state_test.rs b/d-engine-core/src/raft_role/candidate_state_test.rs index 3844f31a..1fac943a 100644 --- a/d-engine-core/src/raft_role/candidate_state_test.rs +++ b/d-engine-core/src/raft_role/candidate_state_test.rs @@ -22,15 +22,15 @@ use crate::ClientCmd; use crate::ConsensusError; use crate::ElectionError; use crate::Error; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MaybeCloneOneshot; use crate::MockBuilder; use crate::MockElectionCore; use crate::MockMembership; use crate::MockReplicationCore; use crate::MockStateMachineHandler; -use crate::RaftEvent; use crate::RaftOneshot; -use crate::RoleEvent; use crate::raft_role::candidate_state::CandidateState; use crate::raft_role::role_state::RaftRoleState; use crate::test_utils::create_test_chunk; @@ -177,7 +177,7 @@ async fn test_tick_triggers_new_election_round_on_success() { // Create new candidate state (term=1) let mut state = CandidateState::::new(1, context.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (event_tx, _event_rx) = mpsc::channel(1); // Advance time to expire the election timer @@ -186,7 +186,7 @@ async fn test_tick_triggers_new_election_round_on_success() { // Execute tick assert!( - state.tick(&role_tx, &event_tx, &context).await.is_ok(), + state.tick(&internal_event_tx, &event_tx, &context).await.is_ok(), "Tick should succeed" ); @@ -241,7 +241,7 @@ async fn test_tick_discovers_higher_term_and_steps_down() { // Create new candidate state (term=1) let mut state = CandidateState::::new(1, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let (event_tx, _event_rx) = mpsc::channel(1); // Advance time to expire the election timer @@ -250,7 +250,7 @@ async fn test_tick_discovers_higher_term_and_steps_down() { // Execute tick assert!( - state.tick(&role_tx, &event_tx, &context).await.is_ok(), + state.tick(&internal_event_tx, &event_tx, &context).await.is_ok(), "Tick should succeed even with HigherTerm error" ); @@ -259,7 +259,10 @@ async fn test_tick_discovers_higher_term_and_steps_down() { // Verify: sent BecomeFollower event assert!( - matches!(role_rx.try_recv().unwrap(), RoleEvent::BecomeFollower(_)), + matches!( + internal_event_rx.try_recv().unwrap(), + InternalEvent::BecomeFollower(_) + ), "Should send BecomeFollower event" ); } @@ -277,7 +280,7 @@ async fn test_tick_discovers_higher_term_and_steps_down() { /// - Term remains unchanged /// /// Original test location: -/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_raft_event_case1_1` +/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_inbound_event_case1_1` #[tokio::test] async fn test_handle_vote_request_rejects_illegal_request() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -296,8 +299,8 @@ async fn test_handle_vote_request_rejects_illegal_request() { // Prepare VoteRequest event let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ReceiveVoteRequest( + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ReceiveVoteRequest( VoteRequest { term: request_term, candidate_id: 1, @@ -309,7 +312,10 @@ async fn test_handle_vote_request_rejects_illegal_request() { // Execute assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), "Should handle event successfully" ); @@ -319,7 +325,7 @@ async fn test_handle_vote_request_rejects_illegal_request() { // Verify: no role change event assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should not send role change event" ); @@ -344,7 +350,7 @@ async fn test_handle_vote_request_rejects_illegal_request() { /// step down and let the new role process it. /// /// Original test location: -/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_raft_event_case1_2` +/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_inbound_event_case1_2` #[tokio::test] async fn test_handle_vote_request_grants_and_steps_down_when_legal() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -362,8 +368,8 @@ async fn test_handle_vote_request_grants_and_steps_down_when_legal() { // Prepare VoteRequest event with higher term let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ReceiveVoteRequest( + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ReceiveVoteRequest( VoteRequest { term: updated_term, candidate_id: 1, @@ -375,19 +381,28 @@ async fn test_handle_vote_request_grants_and_steps_down_when_legal() { // Execute assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), "Should handle event successfully" ); // Verify: steps down to Follower assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(None))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(None)) + ), "Should send BecomeFollower event" ); // Verify: reprocess event assert!( - matches!(role_rx.try_recv().unwrap(), RoleEvent::ReprocessEvent(_)), + matches!( + internal_event_rx.try_recv().unwrap(), + InternalEvent::ReprocessEvent(_) + ), "Should send ReprocessEvent for Follower to handle" ); @@ -418,7 +433,7 @@ async fn test_handle_vote_request_grants_and_steps_down_when_legal() { /// This validates that Candidate can serve read-only cluster metadata queries. /// /// Original test location: -/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_raft_event_case2` +/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_inbound_event_case2` #[tokio::test] async fn test_handle_cluster_conf_metadata_request() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -439,12 +454,15 @@ async fn test_handle_cluster_conf_metadata_request() { // Prepare ClusterConf event let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConf(MetadataRequest {}, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConf(MetadataRequest {}, resp_tx); // Execute assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), "Should handle ClusterConf event" ); @@ -470,7 +488,7 @@ async fn test_handle_cluster_conf_metadata_request() { /// This validates that Candidate can apply configuration changes from leader. /// /// Original test location: -/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_raft_event_case3` +/// `d-engine-server/tests/components/raft_role/candidate_state_test.rs::test_handle_inbound_event_case3` #[tokio::test] async fn test_handle_cluster_conf_update_success() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -497,8 +515,8 @@ async fn test_handle_cluster_conf_update_success() { // Prepare ClusterConfUpdate event let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 2, // Leader ID term: 1, @@ -510,7 +528,10 @@ async fn test_handle_cluster_conf_update_success() { // Execute assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), "Should handle ClusterConfUpdate event" ); @@ -538,7 +559,7 @@ async fn test_handle_cluster_conf_update_success() { /// - Commit index NOT updated (Follower will handle) /// - No response sent (Follower will send it) /// -/// Original: test_handle_raft_event_case4_1 +/// Original: test_handle_inbound_event_case4_1 #[tokio::test] async fn test_handle_append_entries_steps_down_to_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -570,18 +591,23 @@ async fn test_handle_append_entries_steps_down_to_follower() { leader_commit_index: new_leader_commit, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::BecomeFollower(None)) + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(None)) )); assert!(matches!( - role_rx.try_recv().unwrap(), - RoleEvent::ReprocessEvent(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::ReprocessEvent(_) )); assert_eq!(state.current_term(), new_leader_term); assert!(state.commit_index() != new_leader_commit); @@ -590,7 +616,7 @@ async fn test_handle_append_entries_steps_down_to_follower() { /// Test: CandidateState rejects AppendEntries with higher local term /// -/// Original: test_handle_raft_event_case4_2 +/// Original: test_handle_inbound_event_case4_2 #[tokio::test] async fn test_handle_append_entries_rejects_lower_term() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -618,11 +644,16 @@ async fn test_handle_append_entries_rejects_lower_term() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); - assert!(role_rx.try_recv().is_err()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); + assert!(internal_event_rx.try_recv().is_err()); assert_eq!(state.current_term(), term); let response = resp_rx.recv().await.expect("should succeed").unwrap(); @@ -644,7 +675,7 @@ async fn test_handle_append_entries_rejects_lower_term() { /// Rule 1 fires and returns higher_term — the candidate correctly informs the stale leader /// of the current term without stepping down. /// -/// Original: test_handle_raft_event_case4_3 +/// Original: test_handle_inbound_event_case4_3 #[tokio::test] async fn test_handle_append_entries_rejects_stale_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -672,12 +703,17 @@ async fn test_handle_append_entries_rejects_stale_leader() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); assert!( - role_rx.try_recv().is_err(), + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); + assert!( + internal_event_rx.try_recv().is_err(), "Candidate must not change role for stale leader" ); assert_eq!(state.current_term(), term); @@ -735,17 +771,28 @@ async fn test_handle_append_entries_same_term_log_conflict_steps_down() { leader_commit_index: 10, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(None))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(None)) + ), "Candidate must step down when receiving AppendEntries with term >= currentTerm (Raft §5.2)" ); assert!( - matches!(role_rx.try_recv().unwrap(), RoleEvent::ReprocessEvent(_)), + matches!( + internal_event_rx.try_recv().unwrap(), + InternalEvent::ReprocessEvent(_) + ), "Candidate must replay the event so Follower can handle the log conflict" ); @@ -806,17 +853,28 @@ async fn test_handle_append_entries_higher_term_log_conflict_steps_down_and_upda leader_commit_index: 10, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(None))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(None)) + ), "Candidate must step down when receiving AppendEntries with term > currentTerm (Raft §5.2)" ); assert!( - matches!(role_rx.try_recv().unwrap(), RoleEvent::ReprocessEvent(_)), + matches!( + internal_event_rx.try_recv().unwrap(), + InternalEvent::ReprocessEvent(_) + ), "Candidate must replay the event so Follower can handle the log conflict" ); @@ -836,7 +894,7 @@ async fn test_handle_append_entries_higher_term_log_conflict_steps_down_and_upda /// Test: CandidateState handles ClientPropose (write request) /// -/// Original: test_handle_raft_event_case5 +/// Original: test_handle_inbound_event_case5 #[tokio::test] async fn test_handle_client_write_returns_not_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -864,24 +922,24 @@ async fn test_handle_client_write_returns_not_leader() { assert!(err.message().contains("Not leader")); } -/// Test: send_replay_raft_event sends correct events +/// Test: send_replay_inbound_event sends correct events /// -/// Original: test_send_replay_raft_event +/// Original: test_send_replay_inbound_event #[test] fn test_send_become_follower_and_replay_events() { let (_graceful_tx, graceful_rx) = watch::channel(()); let context = mock_raft_context("/tmp/test_replay_event", graceful_rx, None); let state = CandidateState::::new(1, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let (resp_tx, _resp_rx) = MaybeCloneOneshot::new(); - assert!(state.send_become_follower_event(&role_tx).is_ok()); + assert!(state.send_become_follower_event(&internal_event_tx).is_ok()); assert!( state - .send_replay_raft_event( - &role_tx, - RaftEvent::ReceiveVoteRequest( + .send_replay_inbound_event( + &internal_event_tx, + InboundEvent::ReceiveVoteRequest( VoteRequest { term: 1, candidate_id: 1, @@ -895,18 +953,18 @@ fn test_send_become_follower_and_replay_events() { ); assert!(matches!( - role_rx.try_recv().unwrap(), - RoleEvent::BecomeFollower(None) + internal_event_rx.try_recv().unwrap(), + InternalEvent::BecomeFollower(None) )); assert!(matches!( - role_rx.try_recv().unwrap(), - RoleEvent::ReprocessEvent(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::ReprocessEvent(_) )); } /// Test: InstallSnapshotChunk returns PermissionDenied /// -/// Original: test_handle_raft_event_case7 +/// Original: test_handle_inbound_event_case7 #[tokio::test] async fn test_handle_install_snapshot_returns_permission_denied() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -917,9 +975,11 @@ async fn test_handle_install_snapshot_returns_permission_denied() { let (tx, rx) = mpsc::channel(32); tx.send(create_test_chunk(0, b"chunk0", 1, 1, 2)).await.unwrap(); drop(tx); - let raft_event = RaftEvent::InstallSnapshotChunk(rx, resp_tx); + let inbound_event = InboundEvent::InstallSnapshotChunk(rx, resp_tx); - let result = state.handle_raft_event(raft_event, &context, mpsc::unbounded_channel().0).await; + let result = state + .handle_inbound_event(inbound_event, &context, mpsc::unbounded_channel().0) + .await; assert!(result.is_err(), "Expected error"); let response = resp_rx.recv().await.expect("Response should be received"); @@ -932,7 +992,7 @@ async fn test_handle_install_snapshot_returns_permission_denied() { /// Test: DiscoverLeader returns PermissionDenied /// -/// Original: test_handle_raft_event_case11 +/// Original: test_handle_inbound_event_case11 #[tokio::test] async fn test_handle_discover_leader_returns_permission_denied() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -944,9 +1004,11 @@ async fn test_handle_discover_leader_returns_permission_denied() { requester_address: "127.0.0.1:9090".to_string(), }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); - let result = state.handle_raft_event(raft_event, &context, mpsc::unbounded_channel().0).await; + let result = state + .handle_inbound_event(inbound_event, &context, mpsc::unbounded_channel().0) + .await; assert!(result.is_ok(), "Expected Ok"); let response = resp_rx.recv().await.expect("Response should be received"); @@ -970,10 +1032,10 @@ mod snapshot_tests { let (_graceful_tx, graceful_rx) = watch::channel(()); let context = mock_raft_context("/tmp/test_role_violation", graceful_rx, None); let mut state = CandidateState::::new(1, context.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // CreateSnapshotEvent → RoleViolation - let e = state.handle_create_snapshot(&context, &role_tx).await.unwrap_err(); + let e = state.handle_create_snapshot(&context, &internal_event_tx).await.unwrap_err(); assert!(matches!( e, Error::Consensus(ConsensusError::RoleViolation { .. }) @@ -981,7 +1043,11 @@ mod snapshot_tests { // SnapshotCreated → RoleViolation let e = state - .handle_snapshot_created(Err(Error::Fatal("test".to_string())), &context, &role_tx) + .handle_snapshot_created( + Err(Error::Fatal("test".to_string())), + &context, + &internal_event_tx, + ) .await .unwrap_err(); assert!(matches!( @@ -1010,7 +1076,7 @@ mod snapshot_tests { /// /// Expected: /// - Event handler returns Ok() without error - /// - No CreateSnapshotEvent is sent to role_tx (channel remains empty) + /// - No CreateSnapshotEvent is sent to internal_event_tx (channel remains empty) /// - No snapshot file is created /// - Candidate state remains unchanged /// @@ -1028,9 +1094,9 @@ mod snapshot_tests { let mut state = CandidateState::::new(1, context.node_config.clone()); // ApplyCompleted with high index that would normally trigger snapshot - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = state.handle_apply_completed(1000, vec![], &context, &role_tx).await; + let result = state.handle_apply_completed(1000, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds without error assert!( @@ -1039,13 +1105,13 @@ mod snapshot_tests { ); // VERIFY 2: No CreateSnapshotEvent is sent (channel should be empty). - // role_tx is kept alive so that Disconnected cannot be confused with "no event sent". - match role_rx.try_recv() { + // internal_event_tx is kept alive so that Disconnected cannot be confused with "no event sent". + match internal_event_rx.try_recv() { Err(mpsc::error::TryRecvError::Empty) => { // Expected: no event sent — correct behavior } Err(mpsc::error::TryRecvError::Disconnected) => { - panic!("role event channel disconnected unexpectedly"); + panic!("internal event channel disconnected unexpectedly"); } Ok(event) => { panic!( @@ -1053,7 +1119,7 @@ mod snapshot_tests { ); } } - drop(role_tx); + drop(internal_event_tx); // VERIFY 3: Candidate state remains unchanged (no snapshot in progress) // Note: CandidateState doesn't have snapshot_in_progress field because it doesn't handle snapshots @@ -1066,7 +1132,7 @@ mod handle_client_read_request { /// Test: ClientReadRequest with LinearizableRead returns NotLeader /// - /// Original: test_handle_raft_event_case6_1 + /// Original: test_handle_inbound_event_case6_1 #[tokio::test] async fn test_client_read_linearizable_returns_not_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1094,7 +1160,7 @@ mod handle_client_read_request { /// Test: ClientReadRequest with EventualConsistency succeeds /// - /// Original: test_handle_raft_event_case6_2 + /// Original: test_handle_inbound_event_case6_2 #[tokio::test] async fn test_client_read_eventual_consistency_succeeds() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1138,10 +1204,10 @@ mod handle_client_read_request { /// - FatalError event from StateMachine component /// /// # When -/// - Candidate handles FatalError event via handle_raft_event() +/// - Candidate handles FatalError event via handle_inbound_event() /// /// # Then -/// - handle_raft_event() returns Error::Fatal +/// - handle_inbound_event() returns Error::Fatal /// - Error message contains source and error details /// - No role transition events are sent (election is aborted) #[tokio::test] @@ -1156,21 +1222,21 @@ async fn test_candidate_handles_fatal_error_returns_error() { let mut candidate = CandidateState::::new(1, context.node_config.clone()); // Create FatalError event - let fatal_error = RaftEvent::FatalError { + let fatal_error = InboundEvent::FatalError { source: "StateMachine".to_string(), error: "Network failure - cannot persist state".to_string(), }; - // Create role event channel - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + // Create internal event channel + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // Handle the FatalError event - let result = candidate.handle_raft_event(fatal_error, &context, role_tx).await; + let result = candidate.handle_inbound_event(fatal_error, &context, internal_event_tx).await; - // VERIFY 1: handle_raft_event() returns Error::Fatal + // VERIFY 1: handle_inbound_event() returns Error::Fatal assert!( result.is_err(), - "Expected handle_raft_event to return Err, got: {result:?}" + "Expected handle_inbound_event to return Err, got: {result:?}" ); // VERIFY 2: Error is Fatal and contains source information @@ -1184,9 +1250,9 @@ async fn test_candidate_handles_fatal_error_returns_error() { other => panic!("Expected Error::Fatal, got: {other:?}"), } - // VERIFY 3: No role events sent (election is aborted) + // VERIFY 3: No internal events sent (election is aborted) assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No role transition events should be sent during FatalError handling" ); } @@ -1233,12 +1299,12 @@ async fn test_new_leader_initializes_empty_buffers() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Candidate - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); // Transition to Leader (Candidate wins election) - raft.handle_role_event(crate::RoleEvent::BecomeLeader) + raft.handle_internal_event(crate::InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -1266,8 +1332,8 @@ async fn test_new_leader_initializes_empty_buffers() { leader.push_client_cmd(cmd, &raft.ctx); // Flush buffers (verify drain → flush works) - let (role_tx, _role_rx) = tokio::sync::mpsc::unbounded_channel(); - let result = leader.flush_cmd_buffers(&raft.ctx, &role_tx).await; + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); + let result = leader.flush_cmd_buffers(&raft.ctx, &internal_event_tx).await; // Verify: Flush succeeds assert!( diff --git a/d-engine-core/src/raft_role/follower_state.rs b/d-engine-core/src/raft_role/follower_state.rs index 4169bdd2..815e93f2 100644 --- a/d-engine-core/src/raft_role/follower_state.rs +++ b/d-engine-core/src/raft_role/follower_state.rs @@ -33,15 +33,15 @@ use super::role_state::check_and_trigger_snapshot; use crate::ConsensusError; use crate::ElectionCore; use crate::ElectionTimer; +use crate::InboundEvent; +use crate::InternalEvent; use crate::Membership; use crate::NetworkError; use crate::PurgeExecutor; use crate::RaftContext; -use crate::RaftEvent; use crate::RaftLog; use crate::RaftNodeConfig; use crate::Result; -use crate::RoleEvent; use crate::StateMachineHandler; use crate::StateTransitionError; use crate::TypeConfig; @@ -158,8 +158,8 @@ impl RaftRoleState for FollowerState { /// step as Candidate async fn tick( &mut self, - role_tx: &mpsc::UnboundedSender, - _event_tx: &mpsc::Sender, + internal_event_tx: &mpsc::UnboundedSender, + _event_tx: &mpsc::Sender, _ctx: &RaftContext, ) -> Result<()> { if Instant::now() < self.timer.next_deadline() { @@ -171,7 +171,7 @@ impl RaftRoleState for FollowerState { debug!("follower::start_election..."); - role_tx.send(RoleEvent::BecomeCandidate).map_err(|e| { + internal_event_tx.send(InternalEvent::BecomeCandidate).map_err(|e| { error!("Failed to send: {:?}", e); NetworkError::SingalSendFailed(format!("{:?}", e)) })?; @@ -179,17 +179,17 @@ impl RaftRoleState for FollowerState { Ok(()) } - async fn handle_raft_event( + async fn handle_inbound_event( &mut self, - raft_event: RaftEvent, + inbound_event: InboundEvent, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, ) -> Result<()> { let state_snapshot = self.state_snapshot(); let my_term = self.current_term(); - match raft_event { - RaftEvent::ReceiveVoteRequest(vote_request, sender) => { + match inbound_event { + InboundEvent::ReceiveVoteRequest(vote_request, sender) => { let candidate_id = vote_request.candidate_id; let LogId { @@ -256,13 +256,13 @@ impl RaftRoleState for FollowerState { error!("Failed to send: {:?}", e); NetworkError::SingalSendFailed(format!("{:?}", e)) })?; - error("handle_raft_event::RaftEvent::ReceiveVoteRequest", &e); + error("handle_inbound_event::InboundEvent::ReceiveVoteRequest", &e); return Err(e); } } } - RaftEvent::ClusterConf(_metadata_request, sender) => { + InboundEvent::ClusterConf(_metadata_request, sender) => { let cluster_conf = ctx .membership() .retrieve_cluster_membership_config(self.shared_state().current_leader()) @@ -275,7 +275,7 @@ impl RaftRoleState for FollowerState { })?; } - RaftEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { + InboundEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { let current_conf_version = ctx.membership().get_cluster_conf_version().await; let current_leader_id = self.shared_state().current_leader(); @@ -316,18 +316,18 @@ impl RaftRoleState for FollowerState { NetworkError::SingalSendFailed(format!("{:?}", e)) })?; } - RaftEvent::AppendEntries(append_entries_request, sender) => { + InboundEvent::AppendEntries(append_entries_request, sender) => { self.handle_append_entries_request_workflow( append_entries_request, sender, ctx, - role_tx, + internal_event_tx, &state_snapshot, ) .await?; } - RaftEvent::InstallSnapshotChunk(stream, sender) => { + InboundEvent::InstallSnapshotChunk(stream, sender) => { // ack_tx is passed to process_snapshot_stream for per-chunk validation. // In push mode the receiver is never read, so we drain it in a background // task to prevent backpressure: process_snapshot_stream awaits each send, @@ -388,7 +388,7 @@ impl RaftRoleState for FollowerState { } } } - RaftEvent::JoinCluster(_join_request, sender) => { + InboundEvent::JoinCluster(_join_request, sender) => { sender .send(Err(Status::permission_denied( "Follower should not receive JoinCluster event.", @@ -402,15 +402,15 @@ impl RaftRoleState for FollowerState { current_role: "Follower", required_role: "Leader", context: format!( - "Follower node {} receives RaftEvent::JoinCluster", + "Follower node {} receives InboundEvent::JoinCluster", ctx.node_id ), } .into()); } - RaftEvent::DiscoverLeader(request, sender) => { - debug!(?request, "Follower::RaftEvent::DiscoverLeader"); + InboundEvent::DiscoverLeader(request, sender) => { + debug!(?request, "Follower::InboundEvent::DiscoverLeader"); sender .send(Err(Status::permission_denied( "Follower should not response DiscoverLeader event.", @@ -423,8 +423,8 @@ impl RaftRoleState for FollowerState { return Ok(()); } - RaftEvent::StreamSnapshot(_ack_rx, _chunk_tx, startup_tx) => { - debug!("Follower::RaftEvent::StreamSnapshot"); + InboundEvent::StreamSnapshot(_ack_rx, _chunk_tx, startup_tx) => { + debug!("Follower::InboundEvent::StreamSnapshot"); warn!("Follower should not receive StreamSnapshot event."); if let Err(e) = startup_tx.send(Err(Status::failed_precondition("Not the leader"))) { @@ -436,7 +436,7 @@ impl RaftRoleState for FollowerState { return Ok(()); } - RaftEvent::FatalError { source, error } => { + InboundEvent::FatalError { source, error } => { error!("[Follower] Fatal error from {}: {}", source, error); return Err(crate::Error::Fatal(format!( "Fatal error from {source}: {error}" @@ -452,7 +452,7 @@ impl RaftRoleState for FollowerState { last_index: u64, _results: Vec, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> crate::Result<()> { // Per Raft §7: each server takes snapshots independently. check_and_trigger_snapshot( @@ -460,7 +460,7 @@ impl RaftRoleState for FollowerState { Follower as i32, self.current_term(), ctx, - role_tx, + internal_event_tx, ) } @@ -470,7 +470,7 @@ impl RaftRoleState for FollowerState { async fn handle_create_snapshot( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Prevent duplicate snapshot creation if self.snapshot_in_progress.load(Ordering::Acquire) { @@ -482,11 +482,11 @@ impl RaftRoleState for FollowerState { let state_machine_handler = ctx.state_machine_handler().clone(); // Use spawn to perform snapshot creation in the background - let role_tx = role_tx.clone(); + let internal_event_tx = internal_event_tx.clone(); tokio::spawn(async move { let result = state_machine_handler.create_snapshot().await; info!("SnapshotCreated event will be processed in another event thread"); - if let Err(e) = role_tx.send(RoleEvent::SnapshotCreated(result)) { + if let Err(e) = internal_event_tx.send(InternalEvent::SnapshotCreated(result)) { error!("Follower failed to send snapshot creation result: {e:?}"); } }); @@ -502,7 +502,7 @@ impl RaftRoleState for FollowerState { &mut self, result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Reset snapshot_in_progress flag self.snapshot_in_progress.store(false, Ordering::SeqCst); @@ -536,7 +536,7 @@ impl RaftRoleState for FollowerState { } // Stale in-flight events — these are Leader-only operations that may arrive - // after a step-down due to role_tx (unbounded, P2) race with BecomeFollower. + // after a step-down due to internal_event_tx (unbounded, P2) race with BecomeFollower. // Both orderings are valid; Follower silently ignores them rather than erroring. fn handle_log_purge_completed( @@ -549,7 +549,7 @@ impl RaftRoleState for FollowerState { async fn handle_promote_ready_learners( &mut self, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Ok(()) } @@ -557,7 +557,7 @@ impl RaftRoleState for FollowerState { async fn handle_membership_applied( &mut self, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Ok(()) } diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index f3cd572e..e701278c 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -28,6 +28,8 @@ use crate::AppendResponseWithUpdates; use crate::ClientCmd; use crate::Error; use crate::HardState; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MaybeCloneOneshot; use crate::MaybeCloneOneshotSender; use crate::MockElectionCore; @@ -36,10 +38,8 @@ use crate::MockReplicationCore; use crate::MockStateMachineHandler; use crate::NetworkError; use crate::NewCommitData; -use crate::RaftEvent; use crate::RaftLog; use crate::RaftOneshot; -use crate::RoleEvent; use crate::StateUpdate; use crate::SystemError; use crate::raft_role::follower_state::FollowerState; @@ -60,8 +60,8 @@ fn create_vote_request_event( term: u64, candidate_id: u32, resp_tx: MaybeCloneOneshotSender>, -) -> RaftEvent { - RaftEvent::ReceiveVoteRequest( +) -> InboundEvent { + InboundEvent::ReceiveVoteRequest( VoteRequest { term, candidate_id, @@ -236,7 +236,7 @@ async fn test_new_restores_persisted_state_on_restart() { /// - No role change /// - Term unchanged /// -/// Original: test_handle_raft_event_case1_1 +/// Original: test_handle_inbound_event_case1_1 #[tokio::test] async fn test_handle_vote_request_rejects_when_handler_returns_none() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -256,14 +256,19 @@ async fn test_handle_vote_request_rejects_when_handler_returns_none() { let term_before = state.current_term(); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = create_vote_request_event(1, 1, resp_tx); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = create_vote_request_event(1, 1, resp_tx); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let r = resp_rx.recv().await.unwrap().unwrap(); assert!(!r.vote_granted, "Should reject vote"); - assert!(role_rx.try_recv().is_err(), "No role change"); + assert!(internal_event_rx.try_recv().is_err(), "No role change"); assert_eq!(term_before, state.current_term(), "Term unchanged"); } @@ -279,7 +284,7 @@ async fn test_handle_vote_request_rejects_when_handler_returns_none() { /// - Term updated to 100 /// - No role change (stays Follower) /// -/// Original: test_handle_raft_event_case1_2 +/// Original: test_handle_inbound_event_case1_2 #[tokio::test] async fn test_handle_vote_request_grants_and_updates_term() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -306,14 +311,22 @@ async fn test_handle_vote_request_grants_and_updates_term() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = create_vote_request_event(1, 1, resp_tx); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = create_vote_request_event(1, 1, resp_tx); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let r = resp_rx.recv().await.unwrap().unwrap(); assert!(r.vote_granted, "Should grant vote"); - assert!(role_rx.try_recv().is_err(), "Should remain Follower"); + assert!( + internal_event_rx.try_recv().is_err(), + "Should remain Follower" + ); assert_eq!(state.current_term(), updated_term, "Term should update"); } @@ -325,10 +338,10 @@ async fn test_handle_vote_request_grants_and_updates_term() { /// /// Expected: /// - Response with vote_granted=false -/// - handle_raft_event returns Error +/// - handle_inbound_event returns Error /// - Term unchanged /// -/// Original: test_handle_raft_event_case1_3 +/// Original: test_handle_inbound_event_case1_3 #[tokio::test] async fn test_handle_vote_request_returns_error_on_handler_failure() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -347,10 +360,15 @@ async fn test_handle_vote_request_returns_error_on_handler_failure() { let term_before = state.current_term(); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = create_vote_request_event(1, 1, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = create_vote_request_event(1, 1, resp_tx); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_err()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_err() + ); let r = resp_rx.recv().await.unwrap().unwrap(); assert!(!r.vote_granted, "Should reject on error"); @@ -365,7 +383,7 @@ async fn test_handle_vote_request_returns_error_on_handler_failure() { /// Expected: /// - Returns current cluster membership configuration /// -/// Original: test_handle_raft_event_case2 +/// Original: test_handle_inbound_event_case2 #[tokio::test] async fn test_handle_cluster_conf_metadata_request() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -385,10 +403,15 @@ async fn test_handle_cluster_conf_metadata_request() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConf(MetadataRequest {}, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConf(MetadataRequest {}, resp_tx); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let m = resp_rx.recv().await.unwrap().unwrap(); assert_eq!(m.nodes, vec![]); @@ -403,7 +426,7 @@ async fn test_handle_cluster_conf_metadata_request() { /// Expected: /// - Returns success response with error_code=Unspecified /// -/// Original: test_handle_raft_event_case3_1 +/// Original: test_handle_inbound_event_case3_1 #[tokio::test] async fn test_handle_cluster_conf_update_success() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -429,8 +452,8 @@ async fn test_handle_cluster_conf_update_success() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 2, term: 1, @@ -440,7 +463,12 @@ async fn test_handle_cluster_conf_update_success() { resp_tx, ); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.recv().await.unwrap().unwrap(); assert!(response.success); @@ -471,20 +499,20 @@ async fn test_tick_triggers_election_on_timeout() { let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let (event_tx, _event_rx) = mpsc::channel(1); let election_timeout_max = context.node_config.raft.election.election_timeout_max; tokio::time::advance(tokio::time::Duration::from_millis(election_timeout_max + 1)).await; assert!( - state.tick(&role_tx, &event_tx, &context).await.is_ok(), + state.tick(&internal_event_tx, &event_tx, &context).await.is_ok(), "Tick should succeed" ); - let r = role_rx.recv().await.unwrap(); + let r = internal_event_rx.recv().await.unwrap(); assert!( - matches!(r, RoleEvent::BecomeCandidate), + matches!(r, InternalEvent::BecomeCandidate), "Should send BecomeCandidate event on timeout" ); } @@ -506,12 +534,12 @@ async fn test_tick_triggers_election_on_timeout() { /// 3. Updates current_term to leader's term (2) /// 4. Updates commit_index to new value (2) /// 5. Returns AppendEntriesResponse with success=true -/// 6. handle_raft_event returns Ok(()) +/// 6. handle_inbound_event returns Ok(()) /// /// This validates the core Raft rule: Follower accepts entries from /// valid leader and updates its state accordingly. /// -/// Original: test_handle_raft_event_case4_1 +/// Original: test_handle_inbound_event_case4_1 #[tokio::test] async fn test_handle_append_entries_success_from_new_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -555,21 +583,24 @@ async fn test_handle_append_entries_success_from_new_leader() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Action: Handle AppendEntries event assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); // Verify: LeaderDiscovered event sent assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::LeaderDiscovered(5, _) + internal_event_rx.try_recv().unwrap(), + InternalEvent::LeaderDiscovered(5, _) ), "Should send LeaderDiscovered event" ); @@ -577,8 +608,8 @@ async fn test_handle_append_entries_success_from_new_leader() { // Verify: NotifyNewCommitIndex event sent assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::NotifyNewCommitIndex(NewCommitData { + internal_event_rx.try_recv().unwrap(), + InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index: _, role: _, current_term: _ @@ -615,11 +646,11 @@ async fn test_handle_append_entries_success_from_new_leader() { /// 1. No events sent (no role change, no commit update) /// 2. Term unchanged (remains at 2) /// 3. Returns AppendEntriesResponse with is_higher_term=true -/// 4. handle_raft_event returns Ok(()) +/// 4. handle_inbound_event returns Ok(()) /// /// This validates the core Raft rule: Reject requests with stale term. /// -/// Original: test_handle_raft_event_case4_2 +/// Original: test_handle_inbound_event_case4_2 #[tokio::test] async fn test_handle_append_entries_rejects_stale_term() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -652,18 +683,24 @@ async fn test_handle_append_entries_rejects_stale_term() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Action: Handle AppendEntries event assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); // Verify: No events sent - assert!(role_rx.try_recv().is_err(), "Should not send any events"); + assert!( + internal_event_rx.try_recv().is_err(), + "Should not send any events" + ); // Verify: Term unchanged assert_eq!( @@ -691,12 +728,12 @@ async fn test_handle_append_entries_rejects_stale_term() { /// 2. No other events sent (no commit update) /// 3. Term updated to leader's term /// 4. Returns AppendEntriesResponse with success=false -/// 5. handle_raft_event returns Err() +/// 5. handle_inbound_event returns Err() /// /// This validates correct error handling: Leader is recognized, but /// append operation failed and must be retried. /// -/// Original: test_handle_raft_event_case4_3 +/// Original: test_handle_inbound_event_case4_3 #[tokio::test] async fn test_handle_append_entries_with_handler_error() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -729,28 +766,31 @@ async fn test_handle_append_entries_with_handler_error() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Action: Handle AppendEntries event assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_err(), - "handle_raft_event should return error" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_err(), + "handle_inbound_event should return error" ); // Verify: LeaderDiscovered event sent (leader is valid) assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::LeaderDiscovered(5, _) + internal_event_rx.try_recv().unwrap(), + InternalEvent::LeaderDiscovered(5, _) ), "Should send LeaderDiscovered event even on error" ); // Verify: No other events assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No other events should be sent" ); @@ -779,11 +819,11 @@ async fn test_handle_append_entries_with_handler_error() { /// Expected: /// - Returns response with success=false /// - error_code = NOT_LEADER -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates rejection of configuration changes from non-leader nodes. /// -/// Original: test_handle_raft_event_case3_2 +/// Original: test_handle_inbound_event_case3_2 #[tokio::test] async fn test_handle_cluster_conf_update_rejects_non_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -809,8 +849,8 @@ async fn test_handle_cluster_conf_update_rejects_non_leader() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 3, // Non-leader ID term: 1, @@ -820,7 +860,12 @@ async fn test_handle_cluster_conf_update_rejects_non_leader() { resp_tx, ); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.recv().await.unwrap().unwrap(); assert!(!response.success, "Should reject non-leader request"); @@ -841,11 +886,11 @@ async fn test_handle_cluster_conf_update_rejects_non_leader() { /// - Returns response with success=false /// - error_code = VERSION_CONFLICT /// - Response includes current version (5) -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates version conflict detection for configuration changes. /// -/// Original: test_handle_raft_event_case3_3 +/// Original: test_handle_inbound_event_case3_3 #[tokio::test] async fn test_handle_cluster_conf_update_detects_version_conflict() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -871,8 +916,8 @@ async fn test_handle_cluster_conf_update_detects_version_conflict() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 2, term: 1, @@ -882,7 +927,12 @@ async fn test_handle_cluster_conf_update_detects_version_conflict() { resp_tx, ); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.recv().await.unwrap().unwrap(); assert!(!response.success, "Should reject stale version"); @@ -904,11 +954,11 @@ async fn test_handle_cluster_conf_update_detects_version_conflict() { /// - Returns response with success=false /// - error_code = TERM_OUTDATED /// - Response includes current term (5) -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates term checking for configuration changes. /// -/// Original: test_handle_raft_event_case3_4 +/// Original: test_handle_inbound_event_case3_4 #[tokio::test] async fn test_handle_cluster_conf_update_rejects_stale_term() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -935,8 +985,8 @@ async fn test_handle_cluster_conf_update_rejects_stale_term() { state.update_current_term(5); // Follower has higher term let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 2, term: 4, // Stale term @@ -946,7 +996,12 @@ async fn test_handle_cluster_conf_update_rejects_stale_term() { resp_tx, ); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.recv().await.unwrap().unwrap(); assert!(!response.success, "Should reject stale term"); @@ -966,11 +1021,11 @@ async fn test_handle_cluster_conf_update_rejects_stale_term() { /// Expected: /// - Returns response with success=false /// - error_code = INTERNAL_ERROR -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates error handling for internal failures during configuration updates. /// -/// Original: test_handle_raft_event_case3_5 +/// Original: test_handle_inbound_event_case3_5 #[tokio::test] async fn test_handle_cluster_conf_update_handles_internal_error() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -992,8 +1047,8 @@ async fn test_handle_cluster_conf_update_handles_internal_error() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 2, term: 1, @@ -1003,7 +1058,12 @@ async fn test_handle_cluster_conf_update_handles_internal_error() { resp_tx, ); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.recv().await.unwrap().unwrap(); assert!(!response.success, "Should fail on internal error"); @@ -1022,12 +1082,12 @@ async fn test_handle_cluster_conf_update_handles_internal_error() { /// Expected: /// - Returns response with success=false /// - error_code = NOT_LEADER -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates behavior when configuration change is attempted but /// cluster leadership is unknown. /// -/// Original: test_handle_raft_event_case3_6 +/// Original: test_handle_inbound_event_case3_6 #[tokio::test] async fn test_handle_cluster_conf_update_when_leader_unknown() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1053,8 +1113,8 @@ async fn test_handle_cluster_conf_update_when_leader_unknown() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 3, term: 1, @@ -1064,7 +1124,12 @@ async fn test_handle_cluster_conf_update_when_leader_unknown() { resp_tx, ); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.recv().await.unwrap().unwrap(); assert!(!response.success, "Should reject when leader unknown"); @@ -1086,13 +1151,13 @@ async fn test_handle_cluster_conf_update_when_leader_unknown() { /// /// Expected: /// - Returns response with error_code = NOT_LEADER -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// - No state changes /// /// This validates the core Raft rule: Only leader can process writes. /// Follower must redirect client to leader. /// -/// Original: test_handle_raft_event_case5 +/// Original: test_handle_inbound_event_case5 #[tokio::test] async fn test_handle_client_write_request_redirects_to_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1158,12 +1223,12 @@ async fn test_scan_cmd_follower_rejects_with_not_leader() { /// /// Expected: /// - Returns response with error_code = NOT_LEADER -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates that linearizable reads must go through leader to ensure /// consistency guarantees (leader lease or ReadIndex protocol). /// -/// Original: test_handle_raft_event_case6_1 +/// Original: test_handle_inbound_event_case6_1 #[tokio::test] async fn test_handle_client_read_request_linearizable_redirects_to_leader() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1203,12 +1268,12 @@ async fn test_handle_client_read_request_linearizable_redirects_to_leader() { /// - Calls state_machine_handler.read_from_state_machine() /// - Returns response with error_code = SUCCESS /// - Returns data from state machine -/// - handle_raft_event returns Ok(()) +/// - handle_inbound_event returns Ok(()) /// /// This validates that followers can serve eventual consistency reads, /// improving read scalability at the cost of potentially stale data. /// -/// Original: test_handle_raft_event_case6_2 +/// Original: test_handle_inbound_event_case6_2 #[tokio::test] async fn test_handle_client_read_request_eventual_consistency_succeeds() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1260,11 +1325,11 @@ async fn test_handle_join_cluster_rejects_on_follower() { address: "127.0.0.1:9090".to_string(), }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::JoinCluster(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::JoinCluster(request, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Handle JoinCluster event - let result = state.handle_raft_event(raft_event, &context, role_tx).await; + let result = state.handle_inbound_event(inbound_event, &context, internal_event_tx).await; // Verify: Returns error assert!( @@ -1291,13 +1356,13 @@ async fn test_handle_join_cluster_rejects_on_follower() { /// /// Expected: /// - Returns Status error with Code::PermissionDenied -/// - handle_raft_event returns Ok() (not a fatal error) +/// - handle_inbound_event returns Ok() (not a fatal error) /// - No state changes /// /// This validates that follower correctly rejects leader discovery /// when it cannot provide leader information. /// -/// Original: test_handle_raft_event_case11 +/// Original: test_handle_inbound_event_case11 #[tokio::test] async fn test_handle_leader_discovery_rejects_on_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -1311,16 +1376,16 @@ async fn test_handle_leader_discovery_rejects_on_follower() { requester_address: "127.0.0.1:9090".to_string(), }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Handle DiscoverLeader event - let result = state.handle_raft_event(raft_event, &context, role_tx).await; + let result = state.handle_inbound_event(inbound_event, &context, internal_event_tx).await; // Verify: Returns Ok (not a fatal error) assert!( result.is_ok(), - "handle_raft_event should return Ok for DiscoverLeader" + "handle_inbound_event should return Ok for DiscoverLeader" ); // Verify: Response with PermissionDenied @@ -1567,7 +1632,7 @@ mod snapshot_tests { /// /// Protocol scenario: a leader steps down to follower while internal events /// (LogPurgeCompleted, PromoteReadyLearners, StepDownSelfRemoved, MembershipApplied) - /// are still queued in role_rx. All must be silently ignored — returning an error + /// are still queued in internal_event_rx. All must be silently ignored — returning an error /// here would trigger a non-fatal warning log and waste a loop iteration for no reason. /// /// Expected: all return Ok(()) — no panic, no state change. @@ -1578,7 +1643,7 @@ mod snapshot_tests { let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // LogPurgeCompleted: leader-only, but stale events after step-down must not error assert!( @@ -1588,19 +1653,19 @@ mod snapshot_tests { // PromoteReadyLearners: leader-only, same reasoning assert!( - state.handle_promote_ready_learners(&context, &role_tx).await.is_ok(), + state.handle_promote_ready_learners(&context, &internal_event_tx).await.is_ok(), "Stale PromoteReadyLearners should be silently ignored" ); // StepDownSelfRemoved: only leader can self-remove, stale event must not error assert!( - state.handle_self_removed(&role_tx).is_ok(), + state.handle_self_removed(&internal_event_tx).is_ok(), "Stale StepDownSelfRemoved should be silently ignored" ); // MembershipApplied: follower has no cache to refresh — pure no-op assert!( - state.handle_membership_applied(&context, &role_tx).await.is_ok(), + state.handle_membership_applied(&context, &internal_event_tx).await.is_ok(), "MembershipApplied on follower should be a no-op" ); } @@ -1621,10 +1686,10 @@ mod snapshot_tests { let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First trigger — starts background snapshot - let result1 = state.handle_create_snapshot(&context, &role_tx).await; + let result1 = state.handle_create_snapshot(&context, &internal_event_tx).await; assert!( result1.is_ok(), "First handle_create_snapshot should succeed" @@ -1635,7 +1700,7 @@ mod snapshot_tests { ); // Second trigger while first is still running — must be a no-op - let result2 = state.handle_create_snapshot(&context, &role_tx).await; + let result2 = state.handle_create_snapshot(&context, &internal_event_tx).await; assert!( result2.is_ok(), "Second handle_create_snapshot should return Ok (skipped)" @@ -1677,8 +1742,10 @@ mod snapshot_tests { }; let snapshot_result = Ok((metadata, std::path::PathBuf::from("/tmp/snap.bin"))); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let result = state + .handle_snapshot_created(snapshot_result, &context, &internal_event_tx) + .await; assert!(result.is_ok(), "handle_snapshot_created should succeed"); assert!( @@ -1712,8 +1779,10 @@ mod snapshot_tests { let snapshot_result = Err(Error::Fatal("Snapshot creation failed".to_string())); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let result = state + .handle_snapshot_created(snapshot_result, &context, &internal_event_tx) + .await; assert!( result.is_ok(), @@ -1740,10 +1809,10 @@ mod snapshot_tests { let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Phase 1: trigger snapshot - assert!(state.handle_create_snapshot(&context, &role_tx).await.is_ok()); + assert!(state.handle_create_snapshot(&context, &internal_event_tx).await.is_ok()); assert!( state.snapshot_in_progress.load(Ordering::SeqCst), "flag set after create" @@ -1758,14 +1827,19 @@ mod snapshot_tests { checksum: bytes::Bytes::new(), }; let ok_result = Ok((metadata, std::path::PathBuf::from("/tmp/snap1.bin"))); - assert!(state.handle_snapshot_created(ok_result, &context, &role_tx).await.is_ok()); + assert!( + state + .handle_snapshot_created(ok_result, &context, &internal_event_tx) + .await + .is_ok() + ); assert!( !state.snapshot_in_progress.load(Ordering::SeqCst), "flag cleared after created" ); // Phase 3: second snapshot can now be triggered - assert!(state.handle_create_snapshot(&context, &role_tx).await.is_ok()); + assert!(state.handle_create_snapshot(&context, &internal_event_tx).await.is_ok()); assert!( state.snapshot_in_progress.load(Ordering::SeqCst), "flag set again for second snapshot" @@ -1792,7 +1866,7 @@ mod handle_client_read_request { /// /// Expected: /// - Returns response with error_code = NOT_LEADER - /// - handle_raft_event returns Ok() + /// - handle_inbound_event returns Ok() /// /// This validates that follower correctly rejects lease-based reads /// which require leader involvement. @@ -1836,7 +1910,7 @@ mod handle_client_read_request { /// /// Expected: /// - Returns response with error_code = NOT_LEADER - /// - handle_raft_event returns Ok() + /// - handle_inbound_event returns Ok() /// /// This validates that follower respects server default policy /// and redirects when it cannot satisfy the consistency requirement. @@ -1886,7 +1960,7 @@ mod handle_client_read_request { /// Expected: /// - Returns response with error_code = SUCCESS /// - Data served from follower's state machine - /// - handle_raft_event returns Ok() + /// - handle_inbound_event returns Ok() /// /// This validates that followers can serve stale reads when /// eventual consistency is acceptable, improving read scalability. @@ -2043,10 +2117,10 @@ mod handle_client_read_request { /// - FatalError event from StateMachine component /// /// # When -/// - Follower handles FatalError event via handle_raft_event() +/// - Follower handles FatalError event via handle_inbound_event() /// /// # Then -/// - handle_raft_event() returns Error::Fatal +/// - handle_inbound_event() returns Error::Fatal /// - Error message contains source and error details /// - No role transition events are sent #[tokio::test] @@ -2063,21 +2137,21 @@ async fn test_follower_handles_fatal_error_returns_error() { FollowerState::::new(1, context.node_config.clone(), hard_state, Some(0)); // Create FatalError event - let fatal_error = RaftEvent::FatalError { + let fatal_error = InboundEvent::FatalError { source: "StateMachine".to_string(), error: "Disk failure".to_string(), }; - // Create role event channel - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + // Create internal event channel + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // Handle the FatalError event - let result = follower.handle_raft_event(fatal_error, &context, role_tx).await; + let result = follower.handle_inbound_event(fatal_error, &context, internal_event_tx).await; - // VERIFY 1: handle_raft_event() returns Error::Fatal + // VERIFY 1: handle_inbound_event() returns Error::Fatal assert!( result.is_err(), - "Expected handle_raft_event to return Err, got: {result:?}" + "Expected handle_inbound_event to return Err, got: {result:?}" ); // VERIFY 2: Error is Fatal and contains source information @@ -2091,9 +2165,9 @@ async fn test_follower_handles_fatal_error_returns_error() { other => panic!("Expected Error::Fatal, got: {other:?}"), } - // VERIFY 3: No role events sent + // VERIFY 3: No internal events sent assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No role transition events should be sent during FatalError handling" ); } @@ -2110,7 +2184,7 @@ async fn test_follower_handles_fatal_error_returns_error() { /// - State machine handler indicates snapshot should be taken /// /// Expected: -/// - RoleEvent::CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) +/// - InternalEvent::CreateSnapshotEvent is sent directly on internal_event_tx (P2 unbounded) /// - No ReprocessEvent wrapper — direct send eliminates the bounded event_tx deadlock path #[tokio::test] async fn test_apply_completed_triggers_snapshot_when_condition_met() { @@ -2137,10 +2211,10 @@ async fn test_apply_completed_triggers_snapshot_when_condition_met() { let mut follower = FollowerState::::new(1, context.node_config.clone(), hard_state, Some(0)); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // ACTION: Handle ApplyCompleted event - let result = follower.handle_apply_completed(100, vec![], &context, &role_tx).await; + let result = follower.handle_apply_completed(100, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds assert!( @@ -2148,18 +2222,18 @@ async fn test_apply_completed_triggers_snapshot_when_condition_met() { "ApplyCompleted should be handled successfully, got: {result:?}" ); - // VERIFY 2: CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) + // VERIFY 2: CreateSnapshotEvent is sent directly on internal_event_tx (P2 unbounded) // check_and_trigger_snapshot no longer wraps in ReprocessEvent — direct send avoids // the extra round-trip through event_tx that caused the original deadlock risk. - let event = role_rx.try_recv().expect("Should receive snapshot trigger event"); + let event = internal_event_rx.try_recv().expect("Should receive snapshot trigger event"); assert!( - matches!(event, RoleEvent::CreateSnapshotEvent), - "Expected RoleEvent::CreateSnapshotEvent, got: {event:?}" + matches!(event, InternalEvent::CreateSnapshotEvent), + "Expected InternalEvent::CreateSnapshotEvent, got: {event:?}" ); // VERIFY 3: No additional events queued assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should only send one snapshot event" ); } @@ -2201,10 +2275,10 @@ async fn test_apply_completed_does_not_trigger_snapshot_when_condition_not_met() let mut follower = FollowerState::::new(1, context.node_config.clone(), hard_state, Some(0)); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // ACTION: Handle ApplyCompleted event - let result = follower.handle_apply_completed(50, vec![], &context, &role_tx).await; + let result = follower.handle_apply_completed(50, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds assert!( @@ -2214,7 +2288,7 @@ async fn test_apply_completed_does_not_trigger_snapshot_when_condition_not_met() // VERIFY 2: No snapshot event is sent assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should not send snapshot event when condition is not met" ); } @@ -2251,10 +2325,10 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { let mut follower = FollowerState::::new(1, context.node_config.clone(), hard_state, Some(0)); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // ACTION: Handle ApplyCompleted event - let result = follower.handle_apply_completed(100, vec![], &context, &role_tx).await; + let result = follower.handle_apply_completed(100, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds assert!( @@ -2264,7 +2338,7 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { // VERIFY 2: No snapshot event is sent (snapshot disabled in config) assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should not send snapshot event when snapshot is disabled in config" ); } @@ -2422,10 +2496,15 @@ async fn test_follower_acks_immediately_after_memory_write() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); // MemFirst: ACK sent immediately, no waiting for fsync let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); @@ -2463,10 +2542,15 @@ async fn test_follower_acks_immediately_for_heartbeat() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let response = resp_rx.try_recv().expect("Heartbeat ACK must be sent immediately"); assert!(response.unwrap().is_success()); @@ -2512,10 +2596,15 @@ async fn test_follower_commit_index_and_ack_both_sent_immediately() { leader_commit_index: new_commit, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); assert_eq!( state.commit_index(), @@ -2563,15 +2652,15 @@ async fn test_follower_install_snapshot_reports_success_when_apply_succeeds() { FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (tx, rx) = mpsc::channel(32); tx.send(SnapshotChunk::default()).await.unwrap(); drop(tx); state - .handle_raft_event( - RaftEvent::InstallSnapshotChunk(rx, resp_tx), + .handle_inbound_event( + InboundEvent::InstallSnapshotChunk(rx, resp_tx), &context, - role_tx, + internal_event_tx, ) .await .unwrap(); @@ -2631,17 +2720,17 @@ async fn test_follower_install_snapshot_reports_failure_when_apply_fails_after_t FollowerState::::new(1, context.node_config.clone(), None, None); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Follower absorbs the error and continues (does not propagate) let (tx, rx) = mpsc::channel(32); tx.send(SnapshotChunk::default()).await.unwrap(); drop(tx); let _ = state - .handle_raft_event( - RaftEvent::InstallSnapshotChunk(rx, resp_tx), + .handle_inbound_event( + InboundEvent::InstallSnapshotChunk(rx, resp_tx), &context, - role_tx, + internal_event_tx, ) .await; @@ -2701,13 +2790,13 @@ async fn test_follower_cluster_conf_always_exposes_current_leader() { state.shared_state.set_current_leader(3); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); assert!( state - .handle_raft_event( - RaftEvent::ClusterConf(MetadataRequest {}, resp_tx), + .handle_inbound_event( + InboundEvent::ClusterConf(MetadataRequest {}, resp_tx), &context, - role_tx + internal_event_tx ) .await .is_ok() diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 260ea194..71b3934d 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -8,11 +8,13 @@ use super::candidate_state::CandidateState; use super::read_lease::now_ms; use super::role_state::RaftRoleState; use super::role_state::check_and_trigger_snapshot; -use super::role_state::send_replay_raft_event; +use super::role_state::send_replay_inbound_event; use crate::BackgroundSnapshotTransfer; use crate::ConnectionType; use crate::ConsensusError; use crate::Error; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MaybeCloneOneshotSender; use crate::Membership; use crate::MembershipError; @@ -20,7 +22,6 @@ use crate::NetworkError; use crate::PeerUpdate; use crate::PurgeExecutor; use crate::RaftContext; -use crate::RaftEvent; use crate::RaftLog; use crate::RaftNodeConfig; use crate::RaftRequestWithSignal; @@ -31,7 +32,6 @@ use crate::ReplicationError; use crate::ReplicationTimer; use crate::Result; use crate::RetryPolicies; -use crate::RoleEvent; use crate::SnapshotConfig; use crate::StateMachine; use crate::StateMachineHandler; @@ -145,7 +145,7 @@ pub(super) struct PostCommitEntry { /// # Lifecycle /// 1. Action registered when entry appended (e.g., `handle_join_cluster`, `initiate_noop_commit`) /// 2. Action triggered when `commit_index` advances past the entry's index -/// 3. Fired via `drain_commit_actions` → `RoleEvent` → `raft.rs` handler +/// 3. Fired via `drain_commit_actions` → `InternalEvent` → `raft.rs` handler /// /// # Invariants /// - Actions only move forward with commit_index advancement @@ -299,7 +299,7 @@ struct ReplicationWorkerConfig { membership: Arc>, retry_policies: RetryPolicies, response_compress_enabled: bool, - role_event_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, state_machine_handler: Arc>, snapshot_config: SnapshotConfig, } @@ -489,7 +489,7 @@ pub struct LeaderState { /// /// **Lifecycle**: /// 1. Insert: When entry appended (e.g., noop at index 5 → insert(5, LeaderNoop)) - /// 2. Drain: When commit_index ≥ 5 → `drain_commit_actions` → send RoleEvent + /// 2. Drain: When commit_index ≥ 5 → `drain_commit_actions` → send InternalEvent /// 3. Timeout: `tick()` scans deadlines → expired entries get error responses /// /// **Contrast with `pending_client_writes`**: @@ -502,7 +502,7 @@ pub struct LeaderState { write_propose_times: HashMap, /// Per-follower replication worker handles. Key = follower node_id. - /// Workers send AppendEntries via transport and relay results back as RoleEvent::AppendResult. + /// Workers send AppendEntries via transport and relay results back as InternalEvent::AppendResult. /// Dropped on role change (LeaderState drop) → workers exit via channel close. replication_workers: HashMap, @@ -628,10 +628,10 @@ impl RaftRoleState for LeaderState { async fn handle_zombie_detected( &mut self, node_id: u32, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> { - self.handle_zombie_node(node_id, role_tx, ctx).await + self.handle_zombie_node(node_id, internal_event_tx, ctx).await } fn handle_snapshot_push_completed( @@ -733,8 +733,8 @@ impl RaftRoleState for LeaderState { /// Trigger heartbeat now async fn tick( &mut self, - role_tx: &mpsc::UnboundedSender, - _raft_tx: &mpsc::Sender, + internal_event_tx: &mpsc::UnboundedSender, + _raft_tx: &mpsc::Sender, ctx: &RaftContext, ) -> Result<()> { let now = Instant::now(); @@ -743,7 +743,7 @@ impl RaftRoleState for LeaderState { // 1. Stale learner O(1) deadline check — None short-circuits immediately if self.stale_check_deadline.is_some_and(|d| now >= d) - && let Err(e) = self.check_and_purge_stale_front(role_tx, ctx).await + && let Err(e) = self.check_and_purge_stale_front(internal_event_tx, ctx).await { error!("Stale learner check failed: {}", e); } @@ -756,7 +756,7 @@ impl RaftRoleState for LeaderState { // Piggyback pending writes onto heartbeat; if nothing to write, // send an empty AppendEntries to maintain leadership and reset timer. let request = self.propose_buffer.flush(); - self.send_heartbeat_or_batch(request, role_tx, ctx).await?; + self.send_heartbeat_or_batch(request, internal_event_tx, ctx).await?; } // 3. Drain expired pending client writes. @@ -820,7 +820,7 @@ impl RaftRoleState for LeaderState { } if noop_timed_out { warn!("LeaderNoop commit timed out — stepping down"); - let _ = role_tx.send(RoleEvent::BecomeFollower(None)); + let _ = internal_event_tx.send(InternalEvent::BecomeFollower(None)); } Ok(()) @@ -1064,7 +1064,7 @@ impl RaftRoleState for LeaderState { async fn flush_cmd_buffers( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Drain-based: unconditionally flush all buffered commands // No timeout/size checks - drain from channel already collected the batch @@ -1076,18 +1076,18 @@ impl RaftRoleState for LeaderState { // Preserves original high-performance code path for write-heavy workloads if has_writes && !has_reads { if let Some(request) = self.propose_buffer.flush() { - self.process_batch(std::iter::once(request), role_tx, ctx).await?; + self.process_batch(std::iter::once(request), internal_event_tx, ctx).await?; } } // Unified path: mixed write+read or pure read workloads // Merges RPC for 2*RTT → 1*RTT optimization in mixed scenarios else if has_writes || has_reads { - self.unified_write_and_linear_read(ctx, role_tx).await?; + self.unified_write_and_linear_read(ctx, internal_event_tx).await?; } // Process lease reads (immediate, no batching) while let Some((req, sender)) = self.lease_read_queue.pop_front() { - if let Err(e) = self.process_lease_read(req, sender, ctx, role_tx).await { + if let Err(e) = self.process_lease_read(req, sender, ctx, internal_event_tx).await { error!("process_lease_read failed: {:?}", e); } } @@ -1100,25 +1100,25 @@ impl RaftRoleState for LeaderState { Ok(()) } - async fn handle_raft_event( + async fn handle_inbound_event( &mut self, - raft_event: RaftEvent, + inbound_event: InboundEvent, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, ) -> Result<()> { let my_id = self.shared_state.node_id; let my_term = self.current_term(); - match raft_event { + match inbound_event { // Leader receives RequestVote(term=X, candidate=Y) // 1. If X > currentTerm: // - Leader → Follower, currentTerm = X // - Replay event // 2. Else: // - Reply with VoteGranted=false, currentTerm=currentTerm - RaftEvent::ReceiveVoteRequest(vote_request, sender) => { + InboundEvent::ReceiveVoteRequest(vote_request, sender) => { debug!( - "handle_raft_event::RaftEvent::ReceiveVoteRequest: {:?}", + "handle_inbound_event::InboundEvent::ReceiveVoteRequest: {:?}", &vote_request ); @@ -1129,12 +1129,12 @@ impl RaftRoleState for LeaderState { // a concurrent ReadActor could still see the old valid lease (window-period bug). self.shared_state.lease.revoke(); // Step down as Follower - self.send_become_follower_event(None, &role_tx)?; + self.send_become_follower_event(None, &internal_event_tx)?; info!("Leader will not process Vote request, it should let Follower do it."); - send_replay_raft_event( - &role_tx, - RaftEvent::ReceiveVoteRequest(vote_request, sender), + send_replay_inbound_event( + &internal_event_tx, + InboundEvent::ReceiveVoteRequest(vote_request, sender), )?; } else { let last_log_id = @@ -1152,7 +1152,7 @@ impl RaftRoleState for LeaderState { } } - RaftEvent::ClusterConf(_metadata_request, sender) => { + InboundEvent::ClusterConf(_metadata_request, sender) => { // Hide leader ID until noop commits: before noop, the leader has not confirmed // quorum readiness. Exposing current_leader_id too early causes clients to route // to this leader, which then rejects them with LeaderNotReady. @@ -1170,10 +1170,10 @@ impl RaftRoleState for LeaderState { } } - RaftEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { + InboundEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { let current_conf_version = ctx.membership().get_cluster_conf_version().await; debug!(%current_conf_version, ?cluste_conf_change_request, - "handle_raft_event::RaftEvent::ClusterConfUpdate", + "handle_inbound_event::InboundEvent::ClusterConfUpdate", ); // Reject the fake Leader append entries request @@ -1195,21 +1195,24 @@ impl RaftRoleState for LeaderState { my_id ); //TODO: if there is a bug? self.update_current_term(vote_request.term); - self.send_become_follower_event(Some(cluste_conf_change_request.id), &role_tx)?; + self.send_become_follower_event( + Some(cluste_conf_change_request.id), + &internal_event_tx, + )?; info!( "Leader will not process append_entries_request, it should let Follower do it." ); - send_replay_raft_event( - &role_tx, - RaftEvent::ClusterConfUpdate(cluste_conf_change_request, sender), + send_replay_inbound_event( + &internal_event_tx, + InboundEvent::ClusterConfUpdate(cluste_conf_change_request, sender), )?; } } - RaftEvent::AppendEntries(append_entries_request, senders) => { + InboundEvent::AppendEntries(append_entries_request, senders) => { debug!( - "handle_raft_event::RaftEvent::AppendEntries: {:?}", + "handle_inbound_event::InboundEvent::AppendEntries: {:?}", &append_entries_request ); @@ -1233,20 +1236,20 @@ impl RaftRoleState for LeaderState { self.shared_state.lease.revoke(); self.send_become_follower_event( Some(append_entries_request.leader_id), - &role_tx, + &internal_event_tx, )?; info!( "Leader will not process append_entries_request, it should let Follower do it." ); - send_replay_raft_event( - &role_tx, - RaftEvent::AppendEntries(append_entries_request, senders), + send_replay_inbound_event( + &internal_event_tx, + InboundEvent::AppendEntries(append_entries_request, senders), )?; } } - RaftEvent::InstallSnapshotChunk(_streaming, sender) => { + InboundEvent::InstallSnapshotChunk(_streaming, sender) => { sender .send(Err(Status::permission_denied("Not Follower or Learner. "))) .map_err(|e| { @@ -1258,20 +1261,20 @@ impl RaftRoleState for LeaderState { current_role: "Leader", required_role: "Follower or Learner", context: format!( - "Leader node {} receives RaftEvent::InstallSnapshotChunk", + "Leader node {} receives InboundEvent::InstallSnapshotChunk", ctx.node_id ), } .into()); } - RaftEvent::JoinCluster(join_request, sender) => { - debug!(?join_request, "Leader::RaftEvent::JoinCluster"); - self.handle_join_cluster(join_request, sender, ctx, &role_tx).await?; + InboundEvent::JoinCluster(join_request, sender) => { + debug!(?join_request, "Leader::InboundEvent::JoinCluster"); + self.handle_join_cluster(join_request, sender, ctx, &internal_event_tx).await?; } - RaftEvent::DiscoverLeader(request, sender) => { - debug!(?request, "Leader::RaftEvent::DiscoverLeader"); + InboundEvent::DiscoverLeader(request, sender) => { + debug!(?request, "Leader::InboundEvent::DiscoverLeader"); if let Some(meta) = ctx.membership().retrieve_node_meta(my_id).await { let response = LeaderDiscoveryResponse { @@ -1290,8 +1293,8 @@ impl RaftRoleState for LeaderState { panic!("{}", msg); } } - RaftEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx) => { - debug!("Leader::RaftEvent::StreamSnapshot"); + InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx) => { + debug!("Leader::InboundEvent::StreamSnapshot"); // Get the latest snapshot metadata if let Some(metadata) = ctx.state_machine().snapshot_metadata() { @@ -1333,7 +1336,7 @@ impl RaftRoleState for LeaderState { } } - RaftEvent::FatalError { source, error } => { + InboundEvent::FatalError { source, error } => { error!("[Leader] Fatal error from {}: {}", source, error); let fatal_status = || tonic::Status::internal(format!("Node fatal error: {error}")); // Notify all pending write requests @@ -1405,7 +1408,7 @@ impl RaftRoleState for LeaderState { last_index: u64, results: Vec, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { let num_results = results.len(); @@ -1458,7 +1461,13 @@ impl RaftRoleState for LeaderState { } // Check snapshot after SM apply — last_applied is now accurate. - check_and_trigger_snapshot(last_index, Leader as i32, self.current_term(), ctx, role_tx)?; + check_and_trigger_snapshot( + last_index, + Leader as i32, + self.current_term(), + ctx, + internal_event_tx, + )?; trace!( "[Leader-{}] TIMING: process_apply_completed({} results)", @@ -1478,7 +1487,7 @@ impl RaftRoleState for LeaderState { &mut self, durable: u64, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) { let new_commit_index = if self.cluster_metadata.single_voter { // MemFirst single-voter: LogFlushed(durable) is the IO checkpoint. @@ -1507,7 +1516,7 @@ impl RaftRoleState for LeaderState { Leader as i32, self.current_term(), new_commit, - role_tx, + internal_event_tx, ) { error!( ?e, @@ -1517,7 +1526,7 @@ impl RaftRoleState for LeaderState { // Drain pending writes committed via local flush (single-voter or flush path). self.drain_pending_client_writes(new_commit); // Fire post-commit actions (noop confirmation, join responses) for this commit. - self.drain_commit_actions(new_commit, ctx, role_tx).await; + self.drain_commit_actions(new_commit, ctx, internal_event_tx).await; // Single-voter: log flush confirms leadership (leader is the entire quorum). // Refresh lease timestamp after all post-commit work to eliminate the yield-point // race window introduced when drain_commit_actions became async. @@ -1538,7 +1547,7 @@ impl RaftRoleState for LeaderState { follower_id: u32, result: Result, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { let leader_term = self.current_term(); @@ -1590,7 +1599,7 @@ impl RaftRoleState for LeaderState { self.drain_pending_writes_with_error(ErrorCode::TermOutdated); // Revoke lease immediately — window-period fix (see VoteRequest branch). self.shared_state.lease.revoke(); - self.send_become_follower_event(None, role_tx)?; + self.send_become_follower_event(None, internal_event_tx)?; return Err(ReplicationError::HigherTerm(response.term).into()); } @@ -1619,7 +1628,7 @@ impl RaftRoleState for LeaderState { self.drain_pending_writes_with_error(ErrorCode::TermOutdated); // Revoke lease immediately — window-period fix (see VoteRequest branch). self.shared_state.lease.revoke(); - self.send_become_follower_event(None, role_tx)?; + self.send_become_follower_event(None, internal_event_tx)?; return Err(ReplicationError::HigherTerm(term).into()); } return Ok(()); @@ -1657,7 +1666,8 @@ impl RaftRoleState for LeaderState { }) .collect(); if !learner_progress.is_empty() - && let Err(e) = self.check_learner_progress(&learner_progress, ctx, role_tx).await + && let Err(e) = + self.check_learner_progress(&learner_progress, ctx, internal_event_tx).await { error!(?e, "check_learner_progress failed"); } @@ -1675,10 +1685,10 @@ impl RaftRoleState for LeaderState { Leader as i32, self.current_term(), new_commit, - role_tx, + internal_event_tx, )?; self.drain_pending_client_writes(new_commit); - self.drain_commit_actions(new_commit, ctx, role_tx).await; + self.drain_commit_actions(new_commit, ctx, internal_event_tx).await; } // Lease refresh and pending_lease_reads drain are triggered by quorum ACK, @@ -1765,7 +1775,7 @@ impl RaftRoleState for LeaderState { async fn handle_create_snapshot( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Prevent duplicate snapshot creation if self.snapshot_in_progress.load(std::sync::atomic::Ordering::Acquire) { @@ -1777,11 +1787,11 @@ impl RaftRoleState for LeaderState { let state_machine_handler = ctx.state_machine_handler().clone(); // Use spawn to perform snapshot creation in the background - let role_tx = role_tx.clone(); + let internal_event_tx = internal_event_tx.clone(); tokio::spawn(async move { let result = state_machine_handler.create_snapshot().await; info!("SnapshotCreated event will be processed in another event thread"); - if let Err(e) = role_tx.send(RoleEvent::SnapshotCreated(result)) { + if let Err(e) = internal_event_tx.send(InternalEvent::SnapshotCreated(result)) { error!("Failed to send snapshot creation result: {e:?}"); } }); @@ -1796,7 +1806,7 @@ impl RaftRoleState for LeaderState { &mut self, result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { self.snapshot_in_progress.store(false, Ordering::SeqCst); @@ -1833,8 +1843,8 @@ impl RaftRoleState for LeaderState { let purge_executor = ctx.purge_executor(); match purge_executor.execute_purge(scheduled).await { Ok(_) => { - if let Err(e) = - role_tx.send(RoleEvent::LogPurgeCompleted(scheduled)) + if let Err(e) = internal_event_tx + .send(InternalEvent::LogPurgeCompleted(scheduled)) { error!(%e, "Failed to notify purge completion"); } @@ -1856,7 +1866,7 @@ impl RaftRoleState for LeaderState { async fn handle_promote_ready_learners( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // SAFETY: Called from main event loop, no reentrancy issues info!( @@ -1928,7 +1938,9 @@ impl RaftRoleState for LeaderState { ); // Attempt promotion and restore batch on failure - let result = self.safe_batch_promote(promotion_node_ids.clone(), ctx, role_tx).await; + let result = self + .safe_batch_promote(promotion_node_ids.clone(), ctx, internal_event_tx) + .await; if let Err(e) = result { // Restore entries to the front of the queue in reverse order @@ -1963,7 +1975,7 @@ impl RaftRoleState for LeaderState { self.pending_promotions.iter().map(|p| p.node_id).collect::>() ); // Important: Re-send the event to trigger next cycle - role_tx.send(RoleEvent::PromoteReadyLearners).map_err(|e| { + internal_event_tx.send(InternalEvent::PromoteReadyLearners).map_err(|e| { error!("Send PromoteReadyLearners event failed: {e:?}"); NetworkError::SingalSendFailed(format!("{:?}", e)) })?; @@ -1979,7 +1991,7 @@ impl RaftRoleState for LeaderState { async fn handle_membership_applied( &mut self, ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Save old membership for comparison let old_replication_targets = self.cluster_metadata.replication_targets.clone(); @@ -2053,7 +2065,7 @@ impl RaftRoleState for LeaderState { /// Default: warn + Ok(()) (defensive; only Leader should receive this). fn handle_self_removed( &mut self, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Only Leader can propose configuration changes and remove itself // Per Raft protocol: Leader steps down immediately after self-removal @@ -2061,7 +2073,7 @@ impl RaftRoleState for LeaderState { "[Leader-{}] Removed from cluster membership, stepping down to Follower", self.node_id() ); - role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { + internal_event_tx.send(InternalEvent::BecomeFollower(None)).map_err(|e| { error!( "[Leader-{}] Failed to send BecomeFollower after self-removal: {:?}", self.node_id(), @@ -2099,7 +2111,7 @@ impl LeaderState { /// Spawn a long-running replication worker task for the given peer. /// /// The worker runs a FIFO loop: receives AppendEntriesRequests from `task_tx`, - /// calls `transport.send_append_request`, and sends results back via `role_event_tx`. + /// calls `transport.send_append_request`, and sends results back via `internal_event_tx`. /// It exits cleanly when `task_tx` is dropped (LeaderState step-down). fn spawn_worker( peer_id: u32, @@ -2136,7 +2148,7 @@ impl LeaderState { membership, retry_policies, response_compress_enabled, - role_event_tx, + internal_event_tx, state_machine_handler, snapshot_config, } = cfg; @@ -2183,13 +2195,13 @@ impl LeaderState { let recv_broken = stream_broken.clone(); // Spawn recv task to monitor ACKs and detect stream disconnection - let recv_role_event_tx = role_event_tx.clone(); + let recv_internal_event_tx = internal_event_tx.clone(); let recv_handle = tokio::spawn(async move { use futures::StreamExt; while let Some(result) = stream_receiver.next().await { match result { Ok(response) => { - let _ = recv_role_event_tx.send(RoleEvent::AppendResult { + let _ = recv_internal_event_tx.send(InternalEvent::AppendResult { follower_id: peer_id, result: Ok(response), }); @@ -2197,7 +2209,8 @@ impl LeaderState { Err(status) => { warn!(peer_id, "Bidi stream recv error: {:?}", status); recv_broken.store(true, Ordering::Release); - let _ = recv_role_event_tx.send(RoleEvent::PeerStreamError { peer_id }); + let _ = recv_internal_event_tx + .send(InternalEvent::PeerStreamError { peer_id }); break; } } @@ -2226,7 +2239,8 @@ impl LeaderState { if stream_sender.send(request).await.is_err() { warn!(peer_id, "Bidi stream sender closed, reconnecting"); stream_broken.store(true, Ordering::Release); - let _ = role_event_tx.send(RoleEvent::PeerStreamError { peer_id }); + let _ = + internal_event_tx.send(InternalEvent::PeerStreamError { peer_id }); break; } } @@ -2245,7 +2259,7 @@ impl LeaderState { let smh = state_machine_handler.clone(); let m = membership.clone(); let c = snapshot_config.clone(); - let tx = role_event_tx.clone(); + let tx = internal_event_tx.clone(); tokio::spawn(async move { let result = t.send_snapshot(peer_id, metadata, smh, m, c).await; let success = result.is_ok(); @@ -2253,7 +2267,8 @@ impl LeaderState { warn!(peer_id, "Snapshot push failed: {:?}", result); } flag.store(false, Ordering::Release); - let _ = tx.send(RoleEvent::SnapshotPushCompleted { peer_id, success }); + let _ = + tx.send(InternalEvent::SnapshotPushCompleted { peer_id, success }); }); } } @@ -2388,7 +2403,7 @@ impl LeaderState { /// Writes a noop `EntryPayload::noop()` to the local log (no sender → no inline wait), /// then records the expected commit index in `pending_commit_actions` keyed by the /// noop log index. When that index commits, `drain_commit_actions` fires - /// `RoleEvent::NoopCommitted { term }` which the Raft loop handles by calling + /// `InternalEvent::NoopCommitted { term }` which the Raft loop handles by calling /// `on_noop_committed()` and notifying watch listeners. /// /// Returns `Err` only on storage failure (write to raft log). All other paths are @@ -2396,7 +2411,7 @@ impl LeaderState { pub(super) async fn initiate_noop_commit( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { let term = self.current_term(); let deadline = Instant::now() @@ -2428,7 +2443,7 @@ impl LeaderState { wait_for_apply_event: false, }, ctx, - role_tx, + internal_event_tx, ) .await { @@ -2479,12 +2494,12 @@ impl LeaderState { /// pending_commit_actions.insert(5, PostCommitEntry::LeaderNoop{term: 2}); /// /// // T2: Commit advances to 5 - /// drain_commit_actions(5, role_tx); - /// // → Sends RoleEvent::NoopCommitted{term: 2} + /// drain_commit_actions(5, internal_event_tx); + /// // → Sends InternalEvent::NoopCommitted{term: 2} /// // → pending_commit_actions now empty (or has entries > 5) /// /// // T3: Commit stays at 5 (redundant call) - /// drain_commit_actions(5, role_tx); + /// drain_commit_actions(5, internal_event_tx); /// // → No action fired (already drained) /// ``` /// @@ -2495,7 +2510,7 @@ impl LeaderState { &mut self, new_commit: u64, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) { let remaining = if new_commit < u64::MAX { self.pending_commit_actions.split_off(&(new_commit + 1)) @@ -2507,7 +2522,7 @@ impl LeaderState { match entry.action { PostCommitAction::LeaderNoop { term } => { // Call on_noop_committed directly — establishes noop_log_id without - // waiting for a second loop iteration through role_rx (P2). + // waiting for a second loop iteration through internal_event_rx (P2). // Only send the event for notify_leader_change, which lives on Raft. if let Err(e) = self.on_noop_committed(ctx) { warn!( @@ -2515,7 +2530,7 @@ impl LeaderState { "on_noop_committed failed — skipping notify_leader_change" ); } else { - let _ = role_tx.send(RoleEvent::NoopCommitted { term }); + let _ = internal_event_tx.send(InternalEvent::NoopCommitted { term }); } } PostCommitAction::NodeJoin { @@ -2612,7 +2627,7 @@ impl LeaderState { &mut self, raft_request_with_signal: RaftRequestWithSignal, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { debug!( "Leader::execute_request_immediately, request_id: {}", @@ -2621,7 +2636,7 @@ impl LeaderState { // Always bypass buffer for immediate execution (quorum verification) let batch = VecDeque::from([raft_request_with_signal]); - self.process_batch(batch, role_tx, ctx).await?; + self.process_batch(batch, internal_event_tx, ctx).await?; Ok(()) } @@ -2665,7 +2680,7 @@ impl LeaderState { pub async fn process_batch( &mut self, batch: impl IntoIterator, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> { self.timer.reset_replication(); @@ -2677,7 +2692,7 @@ impl LeaderState { trace!(?payloads, "[Node-{}] process_batch", ctx.node_id); } - self.execute_and_process_raft_rpc(payloads, write_metadata, None, ctx, role_tx) + self.execute_and_process_raft_rpc(payloads, write_metadata, None, ctx, internal_event_tx) .await } @@ -2687,7 +2702,7 @@ impl LeaderState { async fn send_heartbeat_or_batch( &mut self, request: Option, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> { self.timer.reset_replication(); @@ -2697,12 +2712,19 @@ impl LeaderState { let start_index = ctx.raft_log().last_entry_id() + 1; let (payloads, write_metadata) = Self::merge_batch_to_write_metadata(std::iter::once(req), start_index); - self.execute_and_process_raft_rpc(payloads, write_metadata, None, ctx, role_tx) - .await + self.execute_and_process_raft_rpc( + payloads, + write_metadata, + None, + ctx, + internal_event_tx, + ) + .await } None => { // No pending writes — send empty AppendEntries as heartbeat. - self.execute_and_process_raft_rpc(vec![], None, None, ctx, role_tx).await + self.execute_and_process_raft_rpc(vec![], None, None, ctx, internal_event_tx) + .await } } } @@ -2756,7 +2778,7 @@ impl LeaderState { &mut self, learner_progress: &HashMap>, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { debug!(?learner_progress, "check_learner_progress"); @@ -2798,7 +2820,7 @@ impl LeaderState { self.node_id(), new_promotions.len() ); - self.enqueue_and_notify_promotions(new_promotions, role_tx)?; + self.enqueue_and_notify_promotions(new_promotions, internal_event_tx)?; } else { trace!( "[NO-PROMOTIONS] Leader {} has no new promotions to enqueue", @@ -2932,7 +2954,7 @@ impl LeaderState { fn enqueue_and_notify_promotions( &mut self, new_promotions: Vec, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { info!( ?new_promotions, @@ -2949,7 +2971,7 @@ impl LeaderState { self.refresh_stale_deadline(threshold); } - role_tx.send(RoleEvent::PromoteReadyLearners).map_err(|e| { + internal_event_tx.send(InternalEvent::PromoteReadyLearners).map_err(|e| { error!("Failed to send PromoteReadyLearners: {e:?}"); Error::System(SystemError::Network(NetworkError::SingalSendFailed( e.to_string(), @@ -3052,16 +3074,18 @@ impl LeaderState { fn send_become_follower_event( &self, new_leader_id: Option, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { info!( ?new_leader_id, "Leader is going to step down as Follower..." ); - role_tx.send(RoleEvent::BecomeFollower(new_leader_id)).map_err(|e| { - error!("Failed to send: {e:?}"); - NetworkError::SingalSendFailed(format!("{:?}", e)) - })?; + internal_event_tx + .send(InternalEvent::BecomeFollower(new_leader_id)) + .map_err(|e| { + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)) + })?; Ok(()) } @@ -3119,7 +3143,7 @@ impl LeaderState { join_request: JoinRequest, sender: MaybeCloneOneshotSender>, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { let node_id = join_request.node_id; let node_role = join_request.node_role; @@ -3167,7 +3191,7 @@ impl LeaderState { wait_for_apply_event: false, }, ctx, - role_tx, + internal_event_tx, ) .await { @@ -3431,15 +3455,15 @@ impl LeaderState { /// Phase 3: fire requests to per-follower workers (fire-and-forget). /// /// Commit and client responses are deferred: - /// - Multi-voter: driven by handle_append_result (RoleEvent::AppendResult from workers) - /// - Single-voter: driven by handle_log_flushed (RoleEvent::LogFlushed from batch_processor) + /// - Multi-voter: driven by handle_append_result (InternalEvent::AppendResult from workers) + /// - Single-voter: driven by handle_log_flushed (InternalEvent::LogFlushed from batch_processor) async fn execute_and_process_raft_rpc( &mut self, payloads: Vec, write_metadata: Option, mut read_batch: Option>, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { trace!( cluster_size = self.cluster_metadata.total_voters, @@ -3586,7 +3610,7 @@ impl LeaderState { membership: membership.clone(), retry_policies: retry_policies.clone(), response_compress_enabled, - role_event_tx: role_tx.clone(), + internal_event_tx: internal_event_tx.clone(), state_machine_handler: state_machine_handler.clone(), snapshot_config: snapshot_config.clone(), }, @@ -3622,7 +3646,7 @@ impl LeaderState { membership: membership.clone(), retry_policies: retry_policies.clone(), response_compress_enabled, - role_event_tx: role_tx.clone(), + internal_event_tx: internal_event_tx.clone(), state_machine_handler: state_machine_handler.clone(), snapshot_config: snapshot_config.clone(), }, @@ -3642,7 +3666,7 @@ impl LeaderState { &mut self, batch: Vec, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { let change = Change::BatchPromote(BatchPromote { node_ids: batch.clone(), @@ -3658,7 +3682,7 @@ impl LeaderState { wait_for_apply_event: false, }, ctx, - role_tx, + internal_event_tx, ) .await?; @@ -3686,7 +3710,7 @@ impl LeaderState { /// so the next stale check is bound to the new front. pub async fn check_and_purge_stale_front( &mut self, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> { let threshold = ctx.node_config.raft.membership.promotion.stale_learner_threshold; @@ -3716,7 +3740,7 @@ impl LeaderState { &[("node_id", node_id.to_string())] ) .increment(1); - if let Err(e) = self.handle_stale_learner(node_id, role_tx, ctx).await { + if let Err(e) = self.handle_stale_learner(node_id, internal_event_tx, ctx).await { error!(node_id, ?e, "Failed to handle stale learner"); } } @@ -3733,7 +3757,7 @@ impl LeaderState { pub async fn handle_zombie_node( &mut self, node_id: u32, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> { let status = ctx.membership().get_node_status(node_id).await; @@ -3756,7 +3780,7 @@ impl LeaderState { pub async fn handle_stale_learner( &mut self, node_id: u32, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> { // Stalled learner detected - remove via membership change (requires consensus) @@ -3778,7 +3802,7 @@ impl LeaderState { wait_for_apply_event: false, }, ctx, - role_tx, + internal_event_tx, ) .await?; @@ -3815,7 +3839,7 @@ impl LeaderState { pub(super) async fn unified_write_and_linear_read( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Extract write metadata let (payloads, write_metadata) = if !self.propose_buffer.is_empty() { @@ -3844,8 +3868,14 @@ impl LeaderState { None }; - self.execute_and_process_raft_rpc(payloads, write_metadata, read_batch, ctx, role_tx) - .await + self.execute_and_process_raft_rpc( + payloads, + write_metadata, + read_batch, + ctx, + internal_event_tx, + ) + .await } /// Execute a batch of reads against the state machine and respond to clients. @@ -3974,7 +4004,7 @@ impl LeaderState { req: ClientReadRequest, sender: MaybeCloneOneshotSender>, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { if self.is_lease_valid() { // Lease valid - serve immediately @@ -4013,7 +4043,8 @@ impl LeaderState { deadline, }); // Fire an empty AppendEntries to trigger lease refresh (fire-and-forget). - self.execute_and_process_raft_rpc(vec![], None, None, ctx, role_tx).await?; + self.execute_and_process_raft_rpc(vec![], None, None, ctx, internal_event_tx) + .await?; } } Ok(()) diff --git a/d-engine-core/src/raft_role/leader_state_test/become_follower_test.rs b/d-engine-core/src/raft_role/leader_state_test/become_follower_test.rs index 4027cd32..62a7ed84 100644 --- a/d-engine-core/src/raft_role/leader_state_test/become_follower_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/become_follower_test.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use tokio::sync::{mpsc, watch}; use crate::RaftNodeConfig; -use crate::event::{RaftEvent, RoleEvent}; +use crate::event::{InboundEvent, InternalEvent}; use crate::maybe_clone_oneshot::{MaybeCloneOneshot, RaftOneshot}; use crate::now_ms; use crate::raft_role::leader_state::LeaderState; @@ -84,8 +84,8 @@ async fn test_receive_higher_term_vote_request_revokes_lease() { ); let (resp_tx, _resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let event = RaftEvent::ReceiveVoteRequest( + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let event = InboundEvent::ReceiveVoteRequest( VoteRequest { term: 999, candidate_id: 2, @@ -94,10 +94,13 @@ async fn test_receive_higher_term_vote_request_revokes_lease() { }, resp_tx, ); - state.handle_raft_event(event, &context, role_tx).await.ok(); + state.handle_inbound_event(event, &context, internal_event_tx).await.ok(); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "higher-term VoteRequest must emit BecomeFollower" ); @@ -133,8 +136,8 @@ async fn test_receive_higher_term_append_entries_revokes_lease() { ); let (resp_tx, _resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let event = RaftEvent::AppendEntries( + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let event = InboundEvent::AppendEntries( AppendEntriesRequest { term: 11, // higher than current term 10 leader_id: 2, @@ -145,10 +148,13 @@ async fn test_receive_higher_term_append_entries_revokes_lease() { }, vec![resp_tx], ); - state.handle_raft_event(event, &context, role_tx).await.ok(); + state.handle_inbound_event(event, &context, internal_event_tx).await.ok(); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "higher-term AppendEntries must emit BecomeFollower" ); @@ -164,11 +170,11 @@ async fn test_receive_higher_term_append_entries_revokes_lease() { /// Higher-term VoteRequest → lease revoked BEFORE Raft loop calls become_follower(). /// /// Pins the window-period safety contract: the Raft loop sends BecomeFollower into -/// role_tx but may not process it immediately (async event-driven). During that gap a +/// internal_event_tx but may not process it immediately (async event-driven). During that gap a /// concurrent ReadActor task can observe the old valid lease and serve a stale LeaseRead. /// /// This test verifies the fix: revoke() is called at the detection point itself, so -/// the lease is invalid as soon as handle_raft_event() returns — before become_follower() +/// the lease is invalid as soon as handle_inbound_event() returns — before become_follower() /// is ever called by the Raft loop. /// /// Expected to FAIL before the fix (lease still valid in window), PASS after. @@ -189,8 +195,8 @@ async fn test_vote_request_higher_term_lease_revoked_before_become_follower() { ); let (resp_tx, _resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let event = RaftEvent::ReceiveVoteRequest( + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let event = InboundEvent::ReceiveVoteRequest( VoteRequest { term: 999, candidate_id: 2, @@ -199,10 +205,13 @@ async fn test_vote_request_higher_term_lease_revoked_before_become_follower() { }, resp_tx, ); - state.handle_raft_event(event, &context, role_tx).await.ok(); + state.handle_inbound_event(event, &context, internal_event_tx).await.ok(); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "higher-term VoteRequest must emit BecomeFollower" ); @@ -211,7 +220,7 @@ async fn test_vote_request_higher_term_lease_revoked_before_become_follower() { assert!( !lease.is_valid(now_ms()), "lease must be revoked at detection point, not waiting for become_follower() \ - — window-period fix required in handle_raft_event VoteRequest branch" + — window-period fix required in handle_inbound_event VoteRequest branch" ); } @@ -237,8 +246,8 @@ async fn test_append_entries_higher_term_lease_revoked_before_become_follower() ); let (resp_tx, _resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let event = RaftEvent::AppendEntries( + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let event = InboundEvent::AppendEntries( AppendEntriesRequest { term: 11, leader_id: 2, @@ -249,10 +258,13 @@ async fn test_append_entries_higher_term_lease_revoked_before_become_follower() }, vec![resp_tx], ); - state.handle_raft_event(event, &context, role_tx).await.ok(); + state.handle_inbound_event(event, &context, internal_event_tx).await.ok(); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "higher-term AppendEntries must emit BecomeFollower" ); @@ -260,7 +272,7 @@ async fn test_append_entries_higher_term_lease_revoked_before_become_follower() assert!( !lease.is_valid(now_ms()), "lease must be revoked at detection point, not waiting for become_follower() \ - — window-period fix required in handle_raft_event AppendEntries branch" + — window-period fix required in handle_inbound_event AppendEntries branch" ); } @@ -283,19 +295,22 @@ async fn test_append_result_higher_term_lease_revoked_before_become_follower() { "precondition: lease must be valid" ); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let higher_term_response = AppendEntriesResponse { node_id: 2, term: 999, result: None, }; let result = state - .handle_append_result(2, Ok(higher_term_response), &context, &role_tx) + .handle_append_result(2, Ok(higher_term_response), &context, &internal_event_tx) .await; assert!(result.is_err(), "higher-term AppendResult must return Err"); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "higher-term AppendResult must emit BecomeFollower" ); @@ -328,19 +343,22 @@ async fn test_append_result_higher_term_revokes_lease() { "precondition: lease must be valid" ); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let higher_term_response = AppendEntriesResponse { node_id: 2, term: 999, // higher than leader term=1 result: None, }; let result = state - .handle_append_result(2, Ok(higher_term_response), &context, &role_tx) + .handle_append_result(2, Ok(higher_term_response), &context, &internal_event_tx) .await; assert!(result.is_err(), "higher-term AppendResult must return Err"); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "higher-term AppendResult must emit BecomeFollower" ); diff --git a/d-engine-core/src/raft_role/leader_state_test/buffer_cleanup_test.rs b/d-engine-core/src/raft_role/leader_state_test/buffer_cleanup_test.rs index 67658549..55d1f2cc 100644 --- a/d-engine-core/src/raft_role/leader_state_test/buffer_cleanup_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/buffer_cleanup_test.rs @@ -1,9 +1,9 @@ use crate::ClientCmd; +use crate::InternalEvent; use crate::MaybeCloneOneshot; use crate::MockBuilder; use crate::RaftOneshot; use crate::RaftRole; -use crate::RoleEvent; use crate::client::ClientWriteRequest; use crate::client::WriteOperation; use crate::raft_role::role_state::RaftRoleState; @@ -52,10 +52,10 @@ async fn test_leader_stepdown_clears_pending_write_buffer() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -89,7 +89,7 @@ async fn test_leader_stepdown_clears_pending_write_buffer() { } // Leader steps down to Follower (receives higher term) - raft.handle_role_event(RoleEvent::BecomeFollower(Some(2))) + raft.handle_internal_event(InternalEvent::BecomeFollower(Some(2))) .await .expect("Should become Follower"); diff --git a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs index f5b6621a..45621053 100644 --- a/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/client_read_test.rs @@ -398,7 +398,7 @@ async fn test_optimization_skip_wait_when_commit_satisfies_read() { /// cannot guarantee linearizability and must reject the read. /// /// # Note -/// Renamed from test_handle_raft_event_case6_1 +/// Renamed from test_handle_inbound_event_case6_1 #[tokio::test] #[traced_test] async fn test_linearizable_read_quorum_failure() { @@ -432,13 +432,13 @@ async fn test_linearizable_read_quorum_failure() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer state.push_client_cmd(cmd, &context); // Flush: triggers quorum verification which will fail - state.flush_cmd_buffers(&context, &role_tx).await.ok(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.ok(); // Then: Client receives error via response channel let e = resp_rx.recv().await.unwrap().unwrap_err(); @@ -476,7 +476,7 @@ async fn test_linearizable_read_quorum_failure() { /// 3. Leader serves read from state machine /// /// # Note -/// Renamed from test_handle_raft_event_case6_2 +/// Renamed from test_handle_inbound_event_case6_2 #[tokio::test] #[traced_test] async fn test_linearizable_read_quorum_success() { @@ -534,20 +534,28 @@ async fn test_linearizable_read_quorum_success() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer state.push_client_cmd(cmd, &context); // Flush: quorum succeeds, read registered in pending_reads[read_index=3] - state.flush_cmd_buffers(&context, &role_tx).await.expect("should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("should succeed"); // Note: In new architecture, commit_index advances asynchronously via handle_append_result // We skip the sync commit assertion and move to ApplyCompleted // Simulate SM apply: ApplyCompleted fires pending_reads for read_index <= 3 state - .handle_apply_completed(expect_new_commit_index, vec![], &context, &role_tx) + .handle_apply_completed( + expect_new_commit_index, + vec![], + &context, + &internal_event_tx, + ) .await .expect("ApplyCompleted should succeed"); @@ -584,7 +592,7 @@ async fn test_linearizable_read_quorum_success() { /// reads after losing leadership. /// /// # Note -/// Renamed from test_handle_raft_event_case6_3 +/// Renamed from test_handle_inbound_event_case6_3 #[tokio::test] #[traced_test] async fn test_linearizable_read_encounters_higher_term() { @@ -630,11 +638,11 @@ async fn test_linearizable_read_encounters_higher_term() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer and flush (triggers leadership verification which discovers higher term) state.push_client_cmd(cmd, &context); - let _ = state.flush_cmd_buffers(&context, &role_tx).await; + let _ = state.flush_cmd_buffers(&context, &internal_event_tx).await; // Then: Leader commit remains unchanged (Fatal error aborted the operation) assert_eq!(state.commit_index(), 1); @@ -713,9 +721,12 @@ async fn test_lease_read_with_valid_lease() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("should succeed"); let response = resp_rx.recv().await.unwrap().unwrap(); assert_eq!(response.error, ErrorCode::Success); @@ -806,9 +817,9 @@ async fn test_expired_lease_single_voter_refreshed_immediately() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.unwrap(); // Lease must be refreshed after flush assert!( @@ -898,14 +909,17 @@ async fn test_unspecified_policy_defaults_to_linearizable_read() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); // Flush: quorum succeeds, read registered in pending_reads[read_index=5] - state.flush_cmd_buffers(&context, &role_tx).await.expect("should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("should succeed"); // Simulate SM apply: releases pending linearizable read state - .handle_apply_completed(5, vec![], &context, &role_tx) + .handle_apply_completed(5, vec![], &context, &internal_event_tx) .await .expect("ApplyCompleted should succeed"); @@ -964,9 +978,12 @@ async fn test_eventual_consistency_serves_immediately() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("should succeed"); let response = resp_rx.recv().await.unwrap().unwrap(); assert_eq!(response.error, ErrorCode::Success); @@ -1024,9 +1041,12 @@ async fn test_server_default_overrides_client_policy() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("should succeed"); let response = resp_rx.recv().await.unwrap().unwrap(); assert_eq!(response.error, ErrorCode::Success); @@ -1083,7 +1103,7 @@ async fn test_linearizable_read_batch_shared_quorum() { membership.expect_replication_peers().returning(Vec::new); state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Push 3 LinearizableRead requests to buffer (simulating drain collection) let req1 = ClientReadRequest { @@ -1111,7 +1131,7 @@ async fn test_linearizable_read_batch_shared_quorum() { state.push_client_cmd(ClientCmd::Read(req3, tx3), &ctx); // Action: Flush buffers (simulating drain-triggered flush) - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Buffer cleared after flush assert_eq!( @@ -1186,7 +1206,7 @@ async fn test_lease_reuse_after_linearizable_read_refresh() { // Verify: Initial lease is invalid assert!(!state.is_lease_valid(), "Lease should be invalid initially"); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action 1: LinearizableRead (triggers quorum + lease refresh) let req1 = ClientReadRequest { @@ -1196,7 +1216,7 @@ async fn test_lease_reuse_after_linearizable_read_refresh() { }; let (tx1, mut rx1) = MaybeCloneOneshot::new(); state.push_client_cmd(ClientCmd::Read(req1, tx1), &ctx); - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Request succeeded assert!(rx1.recv().await.is_ok(), "LinearizableRead should succeed"); @@ -1204,7 +1224,7 @@ async fn test_lease_reuse_after_linearizable_read_refresh() { // Verify: Lease is now valid (refreshed by LinearizableRead) // After flush in single-voter: lease refreshed via handle_log_flushed // Manually trigger to simulate the async flush event - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; assert!( state.is_lease_valid(), @@ -1219,7 +1239,7 @@ async fn test_lease_reuse_after_linearizable_read_refresh() { }; let (tx2, mut rx2) = MaybeCloneOneshot::new(); state.push_client_cmd(ClientCmd::Read(req2, tx2), &ctx); - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Request succeeded (reused lease, no quorum check) assert!( @@ -1280,7 +1300,7 @@ async fn test_eventual_consistency_ignores_stale_lease() { "Lease should be invalid (stale leader)" ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: EventualConsistency read let req = ClientReadRequest { @@ -1290,7 +1310,7 @@ async fn test_eventual_consistency_ignores_stale_lease() { }; let (tx, mut rx) = MaybeCloneOneshot::new(); state.push_client_cmd(ClientCmd::Read(req, tx), &ctx); - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Request succeeded despite stale lease assert!( @@ -1348,7 +1368,7 @@ async fn test_client_policy_override_allowed() { // Setup valid lease so LeaseRead can succeed state.test_update_lease_timestamp(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Client requests LeaseRead (different from server default) let req = ClientReadRequest { @@ -1368,7 +1388,7 @@ async fn test_client_policy_override_allowed() { // Execute: Process the read with client-specified policy state.push_client_cmd(ClientCmd::Read(req, tx), &ctx); - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Request succeeded using LeaseRead (no quorum verification) assert!( @@ -1424,7 +1444,7 @@ async fn test_client_policy_override_denied() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Client requests EventualConsistency (weaker than server default) let req = ClientReadRequest { @@ -1444,7 +1464,7 @@ async fn test_client_policy_override_denied() { // Execute: Process the read with server-enforced policy state.push_client_cmd(ClientCmd::Read(req, tx), &ctx); - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Request succeeded using LinearizableRead (quorum verification performed) assert!( @@ -1507,7 +1527,7 @@ async fn test_drain_single_request_no_delay() { membership.expect_replication_peers().returning(Vec::new); state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Single request (simulates low load) let start = Instant::now(); @@ -1528,7 +1548,7 @@ async fn test_drain_single_request_no_delay() { ); // Flush immediately (drain pattern: no waiting for timeout or size threshold) - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); let elapsed = start.elapsed(); // Verify: Request succeeded @@ -1587,7 +1607,7 @@ async fn test_drain_multiple_requests_natural_batch() { membership.expect_replication_peers().returning(Vec::new); state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Push 10 requests to buffer (simulates high load accumulation) let mut receivers = vec![]; @@ -1610,7 +1630,7 @@ async fn test_drain_multiple_requests_natural_batch() { ); // Action: Single flush processes entire batch - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Buffer emptied (all requests processed together) assert_eq!( @@ -1761,7 +1781,7 @@ async fn test_linearizable_read_batch_single_quorum() { membership.expect_replication_peers().returning(Vec::new); state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Push 5 LinearizableRead requests let mut receivers = vec![]; @@ -1777,7 +1797,7 @@ async fn test_linearizable_read_batch_single_quorum() { } // Action: Flush batch (triggers single quorum verification) - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: All 5 requests succeeded for (i, mut rx) in receivers.into_iter().enumerate() { @@ -1833,7 +1853,7 @@ async fn test_lease_read_valid_lease_serves_immediately() { membership.expect_replication_peers().returning(Vec::new); state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let req = ClientReadRequest { client_id: 1, @@ -1843,7 +1863,7 @@ async fn test_lease_read_valid_lease_serves_immediately() { let (tx, mut rx) = MaybeCloneOneshot::new(); // Action: process_lease_read directly (valid lease → serve immediately, no quorum) - let result = state.process_lease_read(req, tx, &ctx, &role_tx).await; + let result = state.process_lease_read(req, tx, &ctx, &internal_event_tx).await; // Verify: succeeds without error assert!(result.is_ok(), "Should succeed with valid lease"); @@ -1914,9 +1934,9 @@ async fn test_linearizable_read_rejected_when_noop_not_committed() { resp_tx, ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.ok(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.ok(); // After fix: client receives LeaderNotReady (noop not yet committed). // Before fix: client receives Ok (unsafe — no quorum confirmation). @@ -2011,7 +2031,7 @@ async fn test_lease_read_empty_payload_verification_hangs_in_multi_node() { ); assert!(!state.is_lease_valid(), "precondition: lease expired"); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let req = ClientReadRequest { client_id: 1, keys: vec![Bytes::from_static(b"key1")], @@ -2021,7 +2041,7 @@ async fn test_lease_read_empty_payload_verification_hangs_in_multi_node() { // process_lease_read must return immediately (no blocking await on quorum) let start = std::time::Instant::now(); - let result = state.process_lease_read(req, tx, &ctx, &role_tx).await; + let result = state.process_lease_read(req, tx, &ctx, &internal_event_tx).await; let elapsed = start.elapsed(); assert!(result.is_ok(), "process_lease_read must not return Err"); @@ -2157,7 +2177,7 @@ async fn test_linearizable_read_served_without_quorum_in_minority_partition() { "precondition: multi-voter cluster" ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let req = ClientReadRequest { client_id: 1, @@ -2169,7 +2189,7 @@ async fn test_linearizable_read_served_without_quorum_in_minority_partition() { tokio::time::timeout( Duration::from_millis(200), - state.flush_cmd_buffers(&ctx, &role_tx), + state.flush_cmd_buffers(&ctx, &internal_event_tx), ) .await .expect("flush_cmd_buffers must not block") diff --git a/d-engine-core/src/raft_role/leader_state_test/client_write_test.rs b/d-engine-core/src/raft_role/leader_state_test/client_write_test.rs index 0b77eb4b..3c424135 100644 --- a/d-engine-core/src/raft_role/leader_state_test/client_write_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/client_write_test.rs @@ -11,7 +11,7 @@ use crate::PrepareResult; use crate::RaftRequestWithSignal; use crate::client::{ClientWriteRequest, WriteOperation}; use crate::client_command_to_entry_payloads; -use crate::event::{NewCommitData, RoleEvent}; +use crate::event::{InternalEvent, NewCommitData}; use crate::maybe_clone_oneshot::MaybeCloneOneshot; use crate::maybe_clone_oneshot::RaftOneshot; use crate::mock_raft_context; @@ -154,7 +154,7 @@ async fn test_process_raft_request_immediate_execution() { }; use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (tx, rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // When: Execute the write request (fire-and-forget replication in new arch) let result = test_context @@ -173,7 +173,7 @@ async fn test_process_raft_request_immediate_execution() { wait_for_apply_event: true, }, &test_context.raft_context, - &role_tx, + &internal_event_tx, ) .await; @@ -186,7 +186,7 @@ async fn test_process_raft_request_immediate_execution() { test_context.state.cluster_metadata.single_voter = true; test_context .state - .handle_log_flushed(5, &test_context.raft_context, &role_tx) + .handle_log_flushed(5, &test_context.raft_context, &internal_event_tx) .await; // Then: Verify commit index updated to 5 @@ -199,8 +199,8 @@ async fn test_process_raft_request_immediate_execution() { // Then: Verify NotifyNewCommitIndex was sent assert!( matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(NewCommitData { + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index: 5, .. })) @@ -218,7 +218,7 @@ async fn test_process_raft_request_immediate_execution() { succeeded: true, }], &test_context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -259,7 +259,7 @@ async fn test_client_write_waits_for_sm_apply_not_just_commit() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (tx, mut rx) = >::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); test_context .state @@ -271,7 +271,7 @@ async fn test_client_write_waits_for_sm_apply_not_just_commit() { wait_for_apply_event: true, }, &test_context.raft_context, - &role_tx, + &internal_event_tx, ) .await .expect("execute should succeed"); @@ -282,7 +282,7 @@ async fn test_client_write_waits_for_sm_apply_not_just_commit() { test_context.state.cluster_metadata.single_voter = true; test_context .state - .handle_log_flushed(5, &test_context.raft_context, &role_tx) + .handle_log_flushed(5, &test_context.raft_context, &internal_event_tx) .await; // Commit has advanced, but client must NOT have received a response yet. @@ -307,7 +307,7 @@ async fn test_client_write_waits_for_sm_apply_not_just_commit() { succeeded: true, }], &test_context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -386,7 +386,7 @@ async fn test_process_raft_request_two_consecutive_forced_sends() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (tx1, rx1) = >::new(); let (tx2, rx2) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // When: Execute first write request (immediate execution) → pending_client_writes[5] let result1 = test_context @@ -406,7 +406,7 @@ async fn test_process_raft_request_two_consecutive_forced_sends() { wait_for_apply_event: true, }, &test_context.raft_context, - &role_tx, + &internal_event_tx, ) .await; @@ -427,7 +427,7 @@ async fn test_process_raft_request_two_consecutive_forced_sends() { wait_for_apply_event: true, }, &test_context.raft_context, - &role_tx, + &internal_event_tx, ) .await; @@ -439,7 +439,7 @@ async fn test_process_raft_request_two_consecutive_forced_sends() { test_context.state.cluster_metadata.single_voter = true; test_context .state - .handle_log_flushed(6, &test_context.raft_context, &role_tx) + .handle_log_flushed(6, &test_context.raft_context, &internal_event_tx) .await; // Then: Verify commit index updated to 6 @@ -451,7 +451,10 @@ async fn test_process_raft_request_two_consecutive_forced_sends() { // Then: Verify NotifyNewCommitIndex was sent assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::NotifyNewCommitIndex(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) + ), "Expected NotifyNewCommitIndex" ); @@ -468,7 +471,7 @@ async fn test_process_raft_request_two_consecutive_forced_sends() { ]; test_context .state - .handle_apply_completed(6, results, &test_context.raft_context, &role_tx) + .handle_apply_completed(6, results, &test_context.raft_context, &internal_event_tx) .await .unwrap(); @@ -518,7 +521,7 @@ async fn test_process_raft_request_batching_enabled() { use crate::ClientCmd; use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (tx, _rx) = >::new(); - let (_role_tx, _role_rx) = mpsc::unbounded_channel::(); + let (_internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); // When: Push request into buffer (batching — not immediately sent) test_context @@ -575,7 +578,7 @@ async fn test_drain_single_write_no_delay() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Single write request (simulates low load) let start = Instant::now(); @@ -597,14 +600,14 @@ async fn test_drain_single_write_no_delay() { ); // Flush immediately (drain pattern: no waiting) - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); let elapsed = start.elapsed(); // Advance commit via single-voter path (no real peers in test). // MemFirst: set last_entry_id=5 before flush. last_entry_id.store(5, Ordering::Relaxed); state.cluster_metadata.single_voter = true; - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; // Simulate SM apply: resolve pending_write_apply so client gets response { @@ -612,7 +615,10 @@ async fn test_drain_single_write_no_delay() { index: 5, succeeded: true, }]; - state.handle_apply_completed(5, results, &ctx, &role_tx).await.unwrap(); + state + .handle_apply_completed(5, results, &ctx, &internal_event_tx) + .await + .unwrap(); } // Verify: Request succeeded @@ -670,7 +676,7 @@ async fn test_drain_multiple_writes_natural_batch() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Push 10 write requests (simulates high load) let mut receivers = vec![]; @@ -694,7 +700,7 @@ async fn test_drain_multiple_writes_natural_batch() { ); // Action: Single flush processes entire batch - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Verify: Buffer emptied assert_eq!( @@ -707,7 +713,7 @@ async fn test_drain_multiple_writes_natural_batch() { // MemFirst: set last_entry_id=14 before flush. last_entry_id.store(14, Ordering::Relaxed); state.cluster_metadata.single_voter = true; - state.handle_log_flushed(14, &ctx, &role_tx).await; + state.handle_log_flushed(14, &ctx, &internal_event_tx).await; // Simulate SM apply: resolve pending_write_apply so clients get responses { @@ -717,7 +723,10 @@ async fn test_drain_multiple_writes_natural_batch() { succeeded: true, }) .collect(); - state.handle_apply_completed(14, results, &ctx, &role_tx).await.unwrap(); + state + .handle_apply_completed(14, results, &ctx, &internal_event_tx) + .await + .unwrap(); } // Verify: All 10 requests received responses @@ -843,7 +852,7 @@ async fn test_write_batch_single_replication() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Push 5 write requests let mut receivers = vec![]; @@ -860,13 +869,13 @@ async fn test_write_batch_single_replication() { } // Action: Flush batch (triggers single prepare_batch_requests call) - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Advance commit via single-voter path: 5 writes at indices 5..9. // MemFirst: set last_entry_id=9 before flush. last_entry_id.store(9, Ordering::Relaxed); state.cluster_metadata.single_voter = true; - state.handle_log_flushed(9, &ctx, &role_tx).await; + state.handle_log_flushed(9, &ctx, &internal_event_tx).await; // Simulate SM apply: resolve pending_write_apply so clients get responses { @@ -876,7 +885,10 @@ async fn test_write_batch_single_replication() { succeeded: true, }) .collect(); - state.handle_apply_completed(9, results, &ctx, &role_tx).await.unwrap(); + state + .handle_apply_completed(9, results, &ctx, &internal_event_tx) + .await + .unwrap(); } // Verify: All 5 writes succeeded @@ -1068,7 +1080,7 @@ async fn test_client_write_deferred_until_sm_apply() { .build_context(); let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let req = ClientWriteRequest { client_id: 1, @@ -1080,13 +1092,13 @@ async fn test_client_write_deferred_until_sm_apply() { state.push_client_cmd(ClientCmd::Propose(req, tx), &ctx); // Flush: fires prepare_batch_requests, write stored in pending_client_writes - state.flush_cmd_buffers(&ctx, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&ctx, &internal_event_tx).await.unwrap(); // Advance commit via single-voter path: moves write to pending_write_apply (wait_for_apply=true). // MemFirst: set last_entry_id=5 before flush. last_entry_id.store(5, Ordering::Relaxed); state.cluster_metadata.single_voter = true; - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; // CRITICAL: response must NOT arrive before SM apply (wait_for_apply_event = true) assert!( @@ -1099,7 +1111,10 @@ async fn test_client_write_deferred_until_sm_apply() { index: 5, succeeded: true, }]; - state.handle_apply_completed(5, results, &ctx, &role_tx).await.unwrap(); + state + .handle_apply_completed(5, results, &ctx, &internal_event_tx) + .await + .unwrap(); // Response must arrive after apply assert!( @@ -1153,7 +1168,7 @@ async fn test_noop_quorum_check_responds_immediately_without_sm_apply() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.update_commit_index(4).expect("Should succeed"); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Noop payload — used for leadership/quorum verification (wait_for_apply_event = false) let payloads = vec![EntryPayload::noop()]; @@ -1167,7 +1182,7 @@ async fn test_noop_quorum_check_responds_immediately_without_sm_apply() { wait_for_apply_event: false, // KEY: noop path does NOT wait for SM apply }, &ctx, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1177,7 +1192,7 @@ async fn test_noop_quorum_check_responds_immediately_without_sm_apply() { // Since wait_for_apply_event=false, response is sent immediately upon commit (no ApplyCompleted needed) last_entry_id.store(5, Ordering::Relaxed); state.cluster_metadata.single_voter = true; - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; // CRITICAL: response must be available WITHOUT ApplyCompleted // If wait_for_apply_event were true, it would still be in pending_write_apply (awaiting SM apply) @@ -1234,7 +1249,7 @@ async fn test_config_change_quorum_check_responds_immediately_without_sm_apply() let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.update_commit_index(4).expect("Should succeed"); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Config change payload — used for membership change quorum verification (wait_for_apply_event = false) let change = Change::AddNode(AddNode { @@ -1253,7 +1268,7 @@ async fn test_config_change_quorum_check_responds_immediately_without_sm_apply() wait_for_apply_event: false, // KEY: config-change path does NOT wait for SM apply }, &ctx, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1263,7 +1278,7 @@ async fn test_config_change_quorum_check_responds_immediately_without_sm_apply() // Since wait_for_apply_event=false, response is sent immediately upon commit (no ApplyCompleted needed) last_entry_id.store(5, Ordering::Relaxed); state.cluster_metadata.single_voter = true; - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; // CRITICAL: response must be available WITHOUT ApplyCompleted let response = rx @@ -1353,11 +1368,14 @@ async fn test_single_voter_client_write_completes_after_log_flushed() { "must be single_voter for this test" ); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Step 1: Client sends write request let (req, mut client_rx) = write_request(); - state.process_batch(VecDeque::from(vec![req]), &role_tx, &ctx).await.unwrap(); + state + .process_batch(VecDeque::from(vec![req]), &internal_event_tx, &ctx) + .await + .unwrap(); // Step 2: Verify pending_client_writes was populated assert_eq!( @@ -1380,7 +1398,7 @@ async fn test_single_voter_client_write_completes_after_log_flushed() { // Step 4: LogFlushed(1) event arrives — entry is now crash-safe. // MemFirst: set last_entry_id=1 to simulate log having entry 1 in memory. last_entry_id.store(1, Ordering::Relaxed); - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Step 5: Verify commit has advanced to 1 assert_eq!( @@ -1392,8 +1410,8 @@ async fn test_single_voter_client_write_completes_after_log_flushed() { // Step 6: Verify NotifyNewCommitIndex was sent assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::NotifyNewCommitIndex(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::NotifyNewCommitIndex(_) ), "NotifyNewCommitIndex must fire after commit advances" ); @@ -1495,8 +1513,11 @@ async fn test_speculative_next_index_before_receiving_append_result() { }); let (req, _rx) = write_request(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - state.process_batch(VecDeque::from(vec![req]), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + state + .process_batch(VecDeque::from(vec![req]), &internal_event_tx, &ctx) + .await + .unwrap(); assert_eq!( state.next_index(2), Some(follower_2_prev_log_index + 1 + 1), diff --git a/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs b/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs index 20734bde..a7811973 100644 --- a/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs @@ -4,7 +4,7 @@ //! never `last_entry_id` (in-memory). Covers both the single-voter fast path and //! the multi-voter quorum path. Fixed by ticket #329. -use crate::event::RoleEvent; +use crate::event::InternalEvent; use crate::maybe_clone_oneshot::MaybeCloneOneshot; use crate::maybe_clone_oneshot::RaftOneshot; use crate::raft_role::leader_state::LeaderState; @@ -112,8 +112,11 @@ async fn test_single_voter_commit_must_not_advance_before_durable() { .returning(|_, _, _, _, _| Ok(crate::PrepareResult::default())); let (req, _rx) = write_request(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.process_batch(VecDeque::from(vec![req]), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state + .process_batch(VecDeque::from(vec![req]), &internal_event_tx, &ctx) + .await + .unwrap(); // Single-voter: commit driven by handle_log_flushed; durable=0 → no advance yet. // CORRECT: commit must NOT advance — entry is in memory but not crash-safe @@ -123,7 +126,7 @@ async fn test_single_voter_commit_must_not_advance_before_durable() { "CORRECT: single-voter commit must not advance until durable_index catches up" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "CORRECT: NotifyNewCommitIndex must not fire before durable" ); } @@ -154,10 +157,13 @@ async fn test_single_voter_commit_advances_after_durable() { .returning(|_, _, _, _, _| Ok(crate::PrepareResult::default())); let (req, _rx) = write_request(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.process_batch(VecDeque::from(vec![req]), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state + .process_batch(VecDeque::from(vec![req]), &internal_event_tx, &ctx) + .await + .unwrap(); // Single-voter: commit driven by handle_log_flushed; durable=1 → advances. - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; assert_eq!( state.shared_state().commit_index, @@ -165,7 +171,10 @@ async fn test_single_voter_commit_advances_after_durable() { "single-voter commit must advance to durable_index after flush" ); assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::NotifyNewCommitIndex(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) + ), "NotifyNewCommitIndex must fire after commit advances" ); } @@ -232,12 +241,15 @@ async fn test_multi_voter_commit_respects_quorum_result() { }); let (req, _rx) = write_request(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.process_batch(VecDeque::from(vec![req]), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state + .process_batch(VecDeque::from(vec![req]), &internal_event_tx, &ctx) + .await + .unwrap(); // Simulate peer responses: both succeed, but calculate_majority_matched_index returns None. let resp = success_response(1, 1); - state.handle_append_result(2, Ok(resp), &ctx, &role_tx).await.unwrap(); - state.handle_append_result(3, Ok(resp), &ctx, &role_tx).await.unwrap(); + state.handle_append_result(2, Ok(resp), &ctx, &internal_event_tx).await.unwrap(); + state.handle_append_result(3, Ok(resp), &ctx, &internal_event_tx).await.unwrap(); assert_eq!( state.shared_state().commit_index, @@ -245,7 +257,7 @@ async fn test_multi_voter_commit_respects_quorum_result() { "multi-voter commit must not advance when calculate_majority returns None" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "NotifyNewCommitIndex must not fire" ); } @@ -273,13 +285,13 @@ async fn test_leader_single_voter_commit_advances_on_log_flushed() { ) .await; - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Precondition: commit_index still at 0 (process_batch saw durable=0) assert_eq!(state.commit_index(), 0); // Action: LogFlushed(1) — entry 1 is now crash-safe - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: commit_index advanced to 1 assert_eq!( @@ -291,8 +303,8 @@ async fn test_leader_single_voter_commit_advances_on_log_flushed() { // Verify: NotifyNewCommitIndex event sent to state machine assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::NotifyNewCommitIndex(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::NotifyNewCommitIndex(_) ), "NotifyNewCommitIndex must fire after commit advances" ); @@ -316,15 +328,15 @@ async fn test_leader_single_voter_log_flushed_noop_when_already_committed() { // Pre-set commit_index to 1 (already committed) state.update_commit_index(1).unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Action: LogFlushed(1) — durable == commit_index, nothing to do - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: commit_index unchanged, no event fired assert_eq!(state.commit_index(), 1, "commit_index must not change"); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "NotifyNewCommitIndex must NOT fire when durable <= commit_index" ); } @@ -385,11 +397,11 @@ async fn test_update_match_index_only_advances() { state.init_cluster_metadata(&ctx.membership).await.unwrap(); assert!(!state.cluster_metadata.single_voter); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // T1: response match=10 → stored state - .handle_append_result(2, Ok(success_response(1, 10)), &ctx, &role_tx) + .handle_append_result(2, Ok(success_response(1, 10)), &ctx, &internal_event_tx) .await .unwrap(); assert_eq!( @@ -400,7 +412,7 @@ async fn test_update_match_index_only_advances() { // T2: out-of-order stale response match=7 → ignored state - .handle_append_result(2, Ok(success_response(1, 7)), &ctx, &role_tx) + .handle_append_result(2, Ok(success_response(1, 7)), &ctx, &internal_event_tx) .await .unwrap(); assert_eq!( @@ -411,7 +423,7 @@ async fn test_update_match_index_only_advances() { // T3: later response match=12 → advances state - .handle_append_result(2, Ok(success_response(1, 12)), &ctx, &role_tx) + .handle_append_result(2, Ok(success_response(1, 12)), &ctx, &internal_event_tx) .await .unwrap(); assert_eq!( @@ -526,10 +538,10 @@ async fn test_handle_log_flushed_excludes_learner_from_quorum() { state.update_match_index(3, 4).unwrap(); state.update_commit_index(4).unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Action: leader flushes entry 5 to disk - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), @@ -538,8 +550,8 @@ async fn test_handle_log_flushed_excludes_learner_from_quorum() { ); assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::NotifyNewCommitIndex(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::NotifyNewCommitIndex(_) ), "NotifyNewCommitIndex must fire after commit advances" ); @@ -603,11 +615,11 @@ async fn test_handle_append_result_excludes_learner_from_quorum() { state.update_match_index(3, 4).unwrap(); state.update_commit_index(4).unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Action: voter(2) ACKs entry 5 — triggers quorum re-calculation state - .handle_append_result(2, Ok(success_response(1, 5)), &ctx, &role_tx) + .handle_append_result(2, Ok(success_response(1, 5)), &ctx, &internal_event_tx) .await .unwrap(); @@ -618,8 +630,8 @@ async fn test_handle_append_result_excludes_learner_from_quorum() { ); assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::NotifyNewCommitIndex(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::NotifyNewCommitIndex(_) ), "NotifyNewCommitIndex must fire after commit advances" ); @@ -709,13 +721,13 @@ async fn test_multi_voter_commit_advances_when_log_flushed_enables_quorum() { state.update_match_index(2, 1).unwrap(); state.update_match_index(3, 1).unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Precondition: commit still at 0 (durable_index was 0 during process_batch) assert_eq!(state.commit_index(), 0); // Action: LogFlushed(1) — leader's entry is now crash-safe, quorum can be calculated - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: commit_index advanced to 1 (quorum of [1,1,1] satisfied) assert_eq!( @@ -727,8 +739,8 @@ async fn test_multi_voter_commit_advances_when_log_flushed_enables_quorum() { // Verify: NotifyNewCommitIndex event sent to state machine assert!( matches!( - role_rx.try_recv().unwrap(), - RoleEvent::NotifyNewCommitIndex(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::NotifyNewCommitIndex(_) ), "NotifyNewCommitIndex must fire after commit advances" ); @@ -770,13 +782,13 @@ async fn test_multi_voter_log_flushed_no_commit_when_quorum_insufficient() { state.update_match_index(2, 0).unwrap(); state.update_match_index(3, 0).unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Precondition: commit at 0 assert_eq!(state.commit_index(), 0); // Action: LogFlushed(1) — leader durable, but peers haven't replicated yet - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: commit_index must stay at 0 (quorum not satisfied) assert_eq!( @@ -787,7 +799,7 @@ async fn test_multi_voter_log_flushed_no_commit_when_quorum_insufficient() { // Verify: no commit event should be sent assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "NotifyNewCommitIndex must NOT fire when quorum not satisfied" ); } diff --git a/d-engine-core/src/raft_role/leader_state_test/deadline_test.rs b/d-engine-core/src/raft_role/leader_state_test/deadline_test.rs index 0440de55..fe09422f 100644 --- a/d-engine-core/src/raft_role/leader_state_test/deadline_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/deadline_test.rs @@ -33,15 +33,15 @@ type ReadResponse = WriteResponse; async fn setup() -> ( LeaderState, crate::RaftContext, - mpsc::UnboundedSender, - mpsc::Sender, + mpsc::UnboundedSender, + mpsc::Sender, ) { let (_shutdown_tx, shutdown_rx) = watch::channel(()); let ctx = MockBuilder::new(shutdown_rx).build_context(); let state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (raft_tx, _raft_rx) = mpsc::channel(1); - (state, ctx, role_tx, raft_tx) + (state, ctx, internal_event_tx, raft_tx) } fn expired() -> Instant { @@ -95,12 +95,12 @@ fn read_batch( /// Expired write: tick() sends deadline_exceeded and removes the entry. #[tokio::test] async fn test_tick_drains_expired_pending_write() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (meta, mut rx) = write_metadata(expired()); state.pending_client_writes.insert(1, meta); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); // Sender must have received deadline_exceeded. let result = rx.recv().await.expect("channel closed"); @@ -117,12 +117,12 @@ async fn test_tick_drains_expired_pending_write() { /// Live write: tick() leaves the entry untouched and sends nothing. #[tokio::test] async fn test_tick_keeps_non_expired_pending_write() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (meta, mut rx) = write_metadata(far_future()); state.pending_client_writes.insert(1, meta); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); // Channel must still be open (no message sent). assert!( @@ -145,12 +145,12 @@ async fn test_tick_keeps_non_expired_pending_write() { /// Expired read: tick() sends deadline_exceeded and removes the batch. #[tokio::test] async fn test_tick_drains_expired_pending_read() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (batch, mut rx) = read_batch(expired()); state.pending_reads.insert(10, batch); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); let result = rx.recv().await.expect("channel closed"); let status = result.expect_err("expected Err(Status)"); @@ -165,12 +165,12 @@ async fn test_tick_drains_expired_pending_read() { /// Live read: tick() leaves the batch untouched and sends nothing. #[tokio::test] async fn test_tick_keeps_non_expired_pending_read() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (batch, mut rx) = read_batch(far_future()); state.pending_reads.insert(10, batch); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); assert!( rx.try_recv().is_err(), @@ -190,7 +190,7 @@ async fn test_tick_keeps_non_expired_pending_read() { /// Expired lease read: tick() sends deadline_exceeded and removes the entry. #[tokio::test] async fn test_tick_drains_expired_pending_lease_read() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (tx, mut rx) = MaybeCloneOneshot::new(); state.pending_lease_reads.push_back(PendingLeaseRead { @@ -203,7 +203,7 @@ async fn test_tick_drains_expired_pending_lease_read() { deadline: expired(), }); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); let result = rx.recv().await.expect("channel closed"); assert_eq!( @@ -220,7 +220,7 @@ async fn test_tick_drains_expired_pending_lease_read() { /// Live lease read: tick() leaves the entry untouched. #[tokio::test] async fn test_tick_keeps_non_expired_pending_lease_read() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (tx, mut rx) = MaybeCloneOneshot::new(); state.pending_lease_reads.push_back(PendingLeaseRead { @@ -233,7 +233,7 @@ async fn test_tick_keeps_non_expired_pending_lease_read() { deadline: far_future(), }); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); assert!( rx.try_recv().is_err(), @@ -246,7 +246,7 @@ async fn test_tick_keeps_non_expired_pending_lease_read() { /// tick() drains only the expired entry; the live entry survives. #[tokio::test] async fn test_tick_drains_only_expired_lease_reads_in_mixed_queue() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (exp_tx, mut exp_rx) = MaybeCloneOneshot::new(); let (live_tx, mut live_rx) = MaybeCloneOneshot::new(); @@ -270,7 +270,7 @@ async fn test_tick_drains_only_expired_lease_reads_in_mixed_queue() { deadline: far_future(), }); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); assert_eq!( exp_rx.recv().await.expect("channel closed").expect_err("expected Err").code(), @@ -294,7 +294,7 @@ async fn test_tick_drains_only_expired_lease_reads_in_mixed_queue() { /// Expired NodeJoin action: tick() sends deadline_exceeded to the join sender. #[tokio::test] async fn test_tick_drains_expired_node_join_commit_action() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (tx, mut rx) = MaybeCloneOneshot::new(); state.pending_commit_actions.insert( @@ -309,7 +309,7 @@ async fn test_tick_drains_expired_node_join_commit_action() { }, ); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); let result = rx.recv().await.expect("channel closed"); assert_eq!( @@ -326,7 +326,7 @@ async fn test_tick_drains_expired_node_join_commit_action() { /// Live NodeJoin action: tick() leaves it untouched. #[tokio::test] async fn test_tick_keeps_non_expired_node_join_commit_action() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (tx, mut rx) = MaybeCloneOneshot::new(); state.pending_commit_actions.insert( @@ -341,7 +341,7 @@ async fn test_tick_keeps_non_expired_node_join_commit_action() { }, ); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); assert!(rx.try_recv().is_err(), "live NodeJoin must not be drained"); assert_eq!( @@ -351,15 +351,15 @@ async fn test_tick_keeps_non_expired_node_join_commit_action() { ); } -/// Expired LeaderNoop action: tick() emits BecomeFollower to role_tx. +/// Expired LeaderNoop action: tick() emits BecomeFollower to internal_event_tx. /// A noop that times out means quorum cannot be confirmed — leader must step down. #[tokio::test] async fn test_tick_expired_leader_noop_triggers_step_down() { let (_shutdown_tx, shutdown_rx) = watch::channel(()); let ctx = MockBuilder::new(shutdown_rx).build_context(); let mut state = LeaderState::::new(1, ctx.node_config.clone()); - // Keep role_rx alive to receive the BecomeFollower event. - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + // Keep internal_event_rx alive to receive the BecomeFollower event. + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let (raft_tx, _raft_rx) = mpsc::channel(1); state.pending_commit_actions.insert( @@ -370,7 +370,7 @@ async fn test_tick_expired_leader_noop_triggers_step_down() { }, ); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); assert!( state.pending_commit_actions.is_empty(), @@ -378,9 +378,9 @@ async fn test_tick_expired_leader_noop_triggers_step_down() { ); // tick() must have sent BecomeFollower - let event = role_rx.try_recv().expect("expected BecomeFollower event"); + let event = internal_event_rx.try_recv().expect("expected BecomeFollower event"); assert!( - matches!(event, crate::RoleEvent::BecomeFollower(_)), + matches!(event, crate::InternalEvent::BecomeFollower(_)), "expected BecomeFollower, got {:?}", event ); @@ -390,7 +390,7 @@ async fn test_tick_expired_leader_noop_triggers_step_down() { /// tick() drains only the expired one; the live entry survives. #[tokio::test] async fn test_tick_drains_only_expired_writes_in_mixed_map() { - let (mut state, ctx, role_tx, raft_tx) = setup().await; + let (mut state, ctx, internal_event_tx, raft_tx) = setup().await; let (expired_meta, mut expired_rx) = write_metadata(expired()); let (live_meta, mut live_rx) = write_metadata(far_future()); @@ -398,7 +398,7 @@ async fn test_tick_drains_only_expired_writes_in_mixed_map() { state.pending_client_writes.insert(1, expired_meta); state.pending_client_writes.insert(2, live_meta); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); // Expired entry notified. let result = expired_rx.recv().await.expect("channel closed"); diff --git a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs index 8ae3e78d..57a1128d 100644 --- a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs @@ -1,6 +1,6 @@ //! Tests for leader state event handling //! -//! This module tests the `handle_raft_event` method for various Raft events +//! This module tests the `handle_inbound_event` method for various inbound events //! including vote requests, append entries, client operations, and more. use crate::client::WriteOperation; @@ -17,8 +17,8 @@ use crate::MockStateMachineHandler; use crate::client::{ClientReadRequest, ClientWriteRequest}; use crate::config::RaftNodeConfig; use crate::config::ReadConsistencyPolicy; -use crate::event::RaftEvent; -use crate::event::{NewCommitData, RoleEvent}; +use crate::event::InboundEvent; +use crate::event::{InternalEvent, NewCommitData}; use crate::maybe_clone_oneshot::RaftOneshot; use crate::raft_role::leader_state::LeaderState; use crate::role_state::RaftRoleState; @@ -45,11 +45,11 @@ fn create_mock_membership() -> crate::MockMembership { } /// Helper to create VoteRequest event -fn setup_handle_raft_event_case1_params( +fn setup_handle_inbound_event_case1_params( resp_tx: crate::MaybeCloneOneshotSender>, term: u64, -) -> RaftEvent { - RaftEvent::ReceiveVoteRequest( +) -> InboundEvent { + InboundEvent::ReceiveVoteRequest( VoteRequest { term, candidate_id: 1, @@ -98,16 +98,21 @@ async fn test_handle_vote_request_reject_same_term() { // Prepare function params use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = setup_handle_raft_event_case1_params(resp_tx, request_term); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = setup_handle_inbound_event_case1_params(resp_tx, request_term); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); // Receive response with vote_granted = false assert!(!resp_rx.recv().await.unwrap().unwrap().vote_granted); - // No role event receives - assert!(role_rx.try_recv().is_err()); + // No internal event receives + assert!(internal_event_rx.try_recv().is_err()); // Term should not be updated assert_eq!(term_before, state.current_term()); @@ -147,27 +152,27 @@ async fn test_handle_vote_request_step_down_on_higher_term() { // Prepare function params use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = setup_handle_raft_event_case1_params(resp_tx, updated_term); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = setup_handle_inbound_event_case1_params(resp_tx, updated_term); - let r = state.handle_raft_event(raft_event, &context, role_tx).await; + let r = state.handle_inbound_event(inbound_event, &context, internal_event_tx).await; assert!(r.is_ok()); // Step to Follower assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::BecomeFollower(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) )); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::ReprocessEvent(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::ReprocessEvent(_)) )); // Term should be updated assert_eq!(state.current_term(), updated_term); // Make sure this assert is at the end of the test function. - // Because we should wait handle_raft_event fun finish running after the role + // Because we should wait handle_inbound_event fun finish running after the role // events been consumed above. assert!(resp_rx.recv().await.is_err()); } @@ -216,10 +221,15 @@ async fn test_handle_cluster_conf_metadata_request() { // Prepare function params use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConf(MetadataRequest {}, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConf(MetadataRequest {}, resp_tx); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); let m = resp_rx.recv().await.unwrap().unwrap(); assert_eq!(m.nodes, vec![]); @@ -268,10 +278,10 @@ async fn test_handle_cluster_conf_update_reject_stale_term() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let raft_event = RaftEvent::ClusterConfUpdate(request, resp_tx); + let inbound_event = InboundEvent::ClusterConfUpdate(request, resp_tx); state - .handle_raft_event(raft_event, &context, mpsc::unbounded_channel().0) + .handle_inbound_event(inbound_event, &context, mpsc::unbounded_channel().0) .await .unwrap(); @@ -324,19 +334,22 @@ async fn test_handle_cluster_conf_update_step_down_on_higher_term() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate(request, resp_tx); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate(request, resp_tx); - state.handle_raft_event(raft_event, &context, role_tx).await.unwrap(); + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .unwrap(); // Verify step down to follower assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::BecomeFollower(Some(2))) + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(Some(2))) )); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::ReprocessEvent(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::ReprocessEvent(_)) )); assert!(resp_rx.recv().await.is_err()); // Original sender should not get response } @@ -389,13 +402,13 @@ async fn test_handle_append_entries_reject_same_term() { }; use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Execute fun state - .handle_raft_event(raft_event, &context, role_tx) + .handle_inbound_event(inbound_event, &context, internal_event_tx) .await .expect("should succeed"); @@ -453,21 +466,26 @@ async fn test_handle_append_entries_step_down_on_higher_term() { }; use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let raft_event = RaftEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Test fun - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); // Validate criterias: step down as Follower assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::BecomeFollower(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) )); assert!(matches!( - role_rx.try_recv().unwrap(), - RoleEvent::ReprocessEvent(_) + internal_event_rx.try_recv().unwrap(), + InternalEvent::ReprocessEvent(_) )); // Validate no response received @@ -513,7 +531,7 @@ async fn test_handle_client_propose_success() { // New state let mut state = LeaderState::::new(1, context.node_config.clone()); - // Handle raft event + // Handle inbound event use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, _resp_rx) = >::new(); let cmd = ClientCmd::Propose( @@ -526,11 +544,14 @@ async fn test_handle_client_propose_success() { resp_tx, ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer then flush to trigger replication state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("flush should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("flush should succeed"); } /// Test handling ClientReadRequest with linearizable read failure @@ -584,13 +605,13 @@ async fn test_handle_client_read_linearizable_failure() { let (resp_tx, mut resp_rx) = >::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer state.push_client_cmd(cmd, &context); // Flush: triggers quorum verification which will fail - state.flush_cmd_buffers(&context, &role_tx).await.ok(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.ok(); // Client receives error via response channel let e = resp_rx.recv().await.unwrap().unwrap_err(); @@ -675,26 +696,31 @@ async fn test_handle_client_read_linearizable_success() { let (resp_tx, mut resp_rx) = >::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer state.push_client_cmd(cmd, &context); // Flush: prepare_batch_requests fires-and-forgets; read deferred to pending_reads[1] - state.flush_cmd_buffers(&context, &role_tx).await.expect("should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("should succeed"); // Advance commit to 3 via single-voter flush path → fires NotifyNewCommitIndex state.cluster_metadata.single_voter = true; - state.handle_log_flushed(expect_new_commit_index, &context, &role_tx).await; + state + .handle_log_flushed(expect_new_commit_index, &context, &internal_event_tx) + .await; // Validation criteria 1: Leader commit should be updated to: 3 assert_eq!(state.shared_state().commit_index, expect_new_commit_index); - // Validation criteria 3: event "RoleEvent::NotifyNewCommitIndex" should be received - let event = role_rx.try_recv().unwrap(); + // Validation criteria 3: event "InternalEvent::NotifyNewCommitIndex" should be received + let event = internal_event_rx.try_recv().unwrap(); assert!(matches!( event, - RoleEvent::NotifyNewCommitIndex(NewCommitData { + InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index: _expect_new_commit_index, role: _, current_term: _ @@ -703,7 +729,12 @@ async fn test_handle_client_read_linearizable_success() { // Simulate SM apply: ApplyCompleted fires pending_reads for read_index <= 3 state - .handle_apply_completed(expect_new_commit_index, vec![], &context, &role_tx) + .handle_apply_completed( + expect_new_commit_index, + vec![], + &context, + &internal_event_tx, + ) .await .expect("ApplyCompleted should succeed"); @@ -765,11 +796,11 @@ async fn test_handle_client_read_encounters_higher_term() { let (resp_tx, mut resp_rx) = >::new(); let cmd = ClientCmd::Read(client_read_request, resp_tx); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Push to buffer and flush: read deferred to pending_reads[1] state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.ok(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.ok(); // Simulate peer responding with higher term → triggers step-down let higher_term_resp = AppendEntriesResponse { @@ -778,16 +809,16 @@ async fn test_handle_client_read_encounters_higher_term() { result: None, }; state - .handle_append_result(2, Ok(higher_term_resp), &context, &role_tx) + .handle_append_result(2, Ok(higher_term_resp), &context, &internal_event_tx) .await .unwrap_err(); // Expected to return Err(HigherTerm) // Validation criteria 1: Leader commit should remain unchanged assert_eq!(state.shared_state().commit_index, 1); - // Validation criteria 2: event "RoleEvent::BecomeFollower" should be received - let event = role_rx.try_recv().unwrap(); - assert!(matches!(event, RoleEvent::BecomeFollower(None))); + // Validation criteria 2: event "InternalEvent::BecomeFollower" should be received + let event = internal_event_rx.try_recv().unwrap(); + assert!(matches!(event, InternalEvent::BecomeFollower(None))); // Drain pending reads (step-down cleanup) → sends Unavailable to clients state.drain_read_buffer().expect("drain should succeed"); @@ -832,18 +863,23 @@ async fn test_handle_install_snapshot_returns_permission_denied() { let (tx, rx) = mpsc::channel(32); tx.send(create_test_chunk(0, b"chunk0", 1, 1, 2)).await.unwrap(); drop(tx); - let raft_event = RaftEvent::InstallSnapshotChunk(rx, resp_tx); + let inbound_event = InboundEvent::InstallSnapshotChunk(rx, resp_tx); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_err()); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_err() + ); // Validation criteria 1: The response should return an error // Assert that resp_rx receives permission_denied let e = resp_rx.recv().await.unwrap().unwrap_err(); assert!(matches!(e.code(), Code::PermissionDenied)); - // Validation criteria 2: No role event should be triggered - assert!(role_rx.try_recv().is_err()); + // Validation criteria 2: No internal event should be triggered + assert!(internal_event_rx.try_recv().is_err()); } // ============================================================================ @@ -907,9 +943,12 @@ async fn test_drain_read_buffer_clears_pending_reads_on_stepdown() { }, resp_tx, ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(cmd, &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("flush should succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("flush should succeed"); // Verify: read is now in pending_reads, not yet served assert!( @@ -947,8 +986,8 @@ async fn test_drain_read_buffer_clears_pending_reads_on_stepdown() { /// /// Per Raft protocol, after a leader commits its own removal from the cluster /// it must immediately step down to Follower. This test verifies: -/// 1. handle_raft_event returns Ok(()) -/// 2. A BecomeFollower(None) event is sent on role_tx +/// 1. handle_inbound_event returns Ok(()) +/// 2. A BecomeFollower(None) event is sent on internal_event_tx #[tokio::test] async fn test_step_down_self_removed_sends_become_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -957,14 +996,14 @@ async fn test_step_down_self_removed_sends_become_follower() { let node_config = raft_context.node_config(); let mut leader_state = LeaderState::::new(1, node_config); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let result = leader_state.handle_self_removed(&role_tx); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let result = leader_state.handle_self_removed(&internal_event_tx); assert!(result.is_ok(), "StepDownSelfRemoved must return Ok"); - let event = role_rx.try_recv().expect("BecomeFollower event must be sent"); + let event = internal_event_rx.try_recv().expect("BecomeFollower event must be sent"); assert!( - matches!(event, RoleEvent::BecomeFollower(None)), + matches!(event, InternalEvent::BecomeFollower(None)), "Expected BecomeFollower(None), got: {event:?}" ); } @@ -1016,13 +1055,13 @@ async fn test_cluster_conf_hides_leader_id_when_noop_not_committed() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); assert!( state - .handle_raft_event( - RaftEvent::ClusterConf(MetadataRequest {}, resp_tx), + .handle_inbound_event( + InboundEvent::ClusterConf(MetadataRequest {}, resp_tx), &context, - role_tx + internal_event_tx ) .await .is_ok() @@ -1070,13 +1109,13 @@ async fn test_cluster_conf_exposes_leader_id_after_noop_committed() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (resp_tx, mut resp_rx) = >::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); assert!( state - .handle_raft_event( - RaftEvent::ClusterConf(MetadataRequest {}, resp_tx), + .handle_inbound_event( + InboundEvent::ClusterConf(MetadataRequest {}, resp_tx), &context, - role_tx + internal_event_tx ) .await .is_ok() @@ -1140,16 +1179,16 @@ async fn test_cluster_conf_leader_id_transitions_after_noop_commits() { state.shared_state.set_current_leader(1); use crate::maybe_clone_oneshot::MaybeCloneOneshot; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Call 1: noop_log_id = None (pre-noop) let (resp_tx1, mut resp_rx1) = >::new(); assert!( state - .handle_raft_event( - RaftEvent::ClusterConf(MetadataRequest {}, resp_tx1), + .handle_inbound_event( + InboundEvent::ClusterConf(MetadataRequest {}, resp_tx1), &context, - role_tx.clone(), + internal_event_tx.clone(), ) .await .is_ok() @@ -1167,10 +1206,10 @@ async fn test_cluster_conf_leader_id_transitions_after_noop_commits() { let (resp_tx2, mut resp_rx2) = >::new(); assert!( state - .handle_raft_event( - RaftEvent::ClusterConf(MetadataRequest {}, resp_tx2), + .handle_inbound_event( + InboundEvent::ClusterConf(MetadataRequest {}, resp_tx2), &context, - role_tx, + internal_event_tx, ) .await .is_ok() diff --git a/d-engine-core/src/raft_role/leader_state_test/fatal_error_test.rs b/d-engine-core/src/raft_role/leader_state_test/fatal_error_test.rs index f94bab6c..040dc93a 100644 --- a/d-engine-core/src/raft_role/leader_state_test/fatal_error_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/fatal_error_test.rs @@ -5,11 +5,11 @@ use super::super::LeaderState; use crate::ClientCmd; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MockMembership; use crate::MockTypeConfig; -use crate::RaftEvent; use crate::ReadConsistencyPolicy; -use crate::RoleEvent; use crate::client::ClientReadRequest; use crate::maybe_clone_oneshot::MaybeCloneOneshot; use crate::maybe_clone_oneshot::RaftOneshot; @@ -38,10 +38,10 @@ use tracing_test::traced_test; /// - FatalError event from StateMachine component with specific error details /// /// # When -/// - Leader handles FatalError event via handle_raft_event() +/// - Leader handles FatalError event via handle_inbound_event() /// /// # Then -/// - handle_raft_event() returns Error::Fatal +/// - handle_inbound_event() returns Error::Fatal /// - Error message contains source and error details from FatalError event /// - No role transition events are sent (leadership continues until higher level handles error) #[tokio::test] @@ -57,21 +57,21 @@ async fn test_leader_handles_fatal_error_notifies_pending_write_apply() { let mut leader = LeaderState::::new(1, context.node_config.clone()); // Create FatalError event from state machine - let fatal_error = RaftEvent::FatalError { + let fatal_error = InboundEvent::FatalError { source: "StateMachine".to_string(), error: "Disk failure - cannot write to persistent storage".to_string(), }; - // Create role event channel - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + // Create internal event channel + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // Handle the FatalError event - let result = leader.handle_raft_event(fatal_error, &context, role_tx).await; + let result = leader.handle_inbound_event(fatal_error, &context, internal_event_tx).await; - // VERIFY 1: handle_raft_event() returns Error::Fatal + // VERIFY 1: handle_inbound_event() returns Error::Fatal assert!( result.is_err(), - "Expected handle_raft_event to return Err, got: {result:?}" + "Expected handle_inbound_event to return Err, got: {result:?}" ); // VERIFY 2: Error is Fatal and contains source information @@ -89,11 +89,11 @@ async fn test_leader_handles_fatal_error_notifies_pending_write_apply() { other => panic!("Expected Error::Fatal, got: {other:?}"), } - // VERIFY 3: No role events sent (FatalError is handled locally in leader) + // VERIFY 3: No internal events sent (FatalError is handled locally in leader) // The leader stops processing at the fatal error and higher level (Raft core) // will eventually shut down the node assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No role transition events should be sent during FatalError handling" ); } @@ -184,15 +184,15 @@ async fn test_fatal_error_drains_all_pending_queues() { ); // -- Action: FatalError -- - let (role_tx, _role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); let result = leader - .handle_raft_event( - RaftEvent::FatalError { + .handle_inbound_event( + InboundEvent::FatalError { source: "StateMachine".to_string(), error: "disk failure".to_string(), }, &ctx, - role_tx, + internal_event_tx, ) .await; diff --git a/d-engine-core/src/raft_role/leader_state_test/lease_refresh_on_log_flushed_test.rs b/d-engine-core/src/raft_role/leader_state_test/lease_refresh_on_log_flushed_test.rs index 2adc6d43..e2145dad 100644 --- a/d-engine-core/src/raft_role/leader_state_test/lease_refresh_on_log_flushed_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/lease_refresh_on_log_flushed_test.rs @@ -151,7 +151,7 @@ async fn setup_multi_voter_leader( async fn test_single_voter_lease_refreshed_on_log_flushed() { let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_single_voter_lease_refresh", 1000).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Precondition: Lease is invalid initially assert!( @@ -162,7 +162,7 @@ async fn test_single_voter_lease_refreshed_on_log_flushed() { // Action: Simulate log flush with durable=1 (commit advances from 0 to 1). // MemFirst: set last_entry_id=1 before flush (log has entry 1, IO confirms it). last_entry_id.store(1, Ordering::Relaxed); - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: Lease is now valid assert!( @@ -194,14 +194,14 @@ async fn test_single_voter_lease_expires_and_refreshes() { let lease_duration_ms = 100; // Short duration for testing let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_single_voter_lease_expiry", lease_duration_ms).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // T0: Lease invalid assert!(!state.is_lease_valid()); // T1: Log flush → lease valid last_entry_id.store(1, Ordering::Relaxed); - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; assert!(state.is_lease_valid()); // T2: Wait for lease to expire @@ -213,7 +213,7 @@ async fn test_single_voter_lease_expires_and_refreshes() { // T3: Another log flush → lease valid again last_entry_id.store(2, Ordering::Relaxed); - state.handle_log_flushed(2, &ctx, &role_tx).await; + state.handle_log_flushed(2, &ctx, &internal_event_tx).await; assert!( state.is_lease_valid(), "Lease should be refreshed by subsequent log flush" @@ -240,14 +240,14 @@ async fn test_single_voter_lease_expires_and_refreshes() { async fn test_single_voter_lease_not_refreshed_when_commit_unchanged() { let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_single_voter_noop_flush", 1000).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Precondition: commit=0, lease invalid assert_eq!(state.commit_index(), 0); assert!(!state.is_lease_valid()); // Action: Flush with durable=0, last_entry_id=0 (no new entries, commit unchanged) - state.handle_log_flushed(0, &ctx, &role_tx).await; + state.handle_log_flushed(0, &ctx, &internal_event_tx).await; // Verify: Lease remains invalid (no leadership proof) assert!(!state.is_lease_valid()); @@ -255,7 +255,7 @@ async fn test_single_voter_lease_not_refreshed_when_commit_unchanged() { // Action: Flush with durable=1 (commit advances); set last_entry_id=1 to match last_entry_id.store(1, Ordering::Relaxed); - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: Now lease is valid assert!(state.is_lease_valid()); @@ -285,12 +285,12 @@ async fn test_single_voter_continuous_flushes_maintain_lease() { let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_single_voter_continuous_flush", lease_duration_ms) .await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Simulate continuous write activity for i in 1..=10u64 { last_entry_id.store(i, Ordering::Relaxed); - state.handle_log_flushed(i, &ctx, &role_tx).await; + state.handle_log_flushed(i, &ctx, &internal_event_tx).await; assert!( state.is_lease_valid(), "Lease should remain valid after flush {i}" @@ -327,7 +327,7 @@ async fn test_single_voter_continuous_flushes_maintain_lease() { async fn test_multi_voter_lease_not_refreshed_on_log_flushed() { let (mut state, ctx) = setup_multi_voter_leader("/tmp/test_multi_voter_no_lease_refresh", 1000).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Precondition: Multi-voter cluster, lease invalid assert!(!state.cluster_metadata.single_voter); @@ -336,7 +336,7 @@ async fn test_multi_voter_lease_not_refreshed_on_log_flushed() { // Action: Log flush with durable=1 // Note: In multi-voter, commit won't advance without follower ACKs, // but we're testing that even if it did, lease wouldn't refresh here. - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; // Verify: Lease remains invalid (must wait for handle_append_result) assert!( @@ -369,13 +369,13 @@ async fn test_multi_voter_lease_not_refreshed_on_log_flushed() { async fn test_multi_voter_lease_refresh_requires_append_result() { let (mut state, ctx) = setup_multi_voter_leader("/tmp/test_multi_voter_append_result_lease", 1000).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Precondition: Lease invalid assert!(!state.is_lease_valid()); // Action 1: Log flush → no lease refresh - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; assert!(!state.is_lease_valid()); // Action 2: Simulate append_result path (via test helper) @@ -407,11 +407,11 @@ async fn test_multi_voter_lease_refresh_requires_append_result() { async fn test_single_voter_durable_regression_does_not_refresh_lease() { let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_durable_regression", 1000).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // T1: Flush(5) → commit=5, lease valid last_entry_id.store(5, Ordering::Relaxed); - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; assert_eq!(state.commit_index(), 5); assert!(state.is_lease_valid()); @@ -420,7 +420,7 @@ async fn test_single_voter_durable_regression_does_not_refresh_lease() { // T2: Flush(3) → should be no-op (last_entry_id=5 > commit=5 is false, no advance) // last_entry_id stays at 5; durable regressing to 3 is the scenario. - state.handle_log_flushed(3, &ctx, &role_tx).await; + state.handle_log_flushed(3, &ctx, &internal_event_tx).await; // Verify: Commit unchanged, lease not updated (timestamp same as T1) assert_eq!(state.commit_index(), 5, "Commit should not regress"); @@ -446,14 +446,14 @@ async fn test_single_voter_very_short_lease_expires_quickly() { let lease_duration_ms = 50; // Short but stable — avoids 1ms clock-boundary race let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_very_short_lease", lease_duration_ms).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // T0: Lease invalid assert!(!state.is_lease_valid()); // T1: Flush updates lease last_entry_id.store(1, Ordering::Relaxed); - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; assert!( state.is_lease_valid(), "Lease should be valid immediately after flush" @@ -483,11 +483,11 @@ async fn test_single_voter_large_lease_duration() { let lease_duration_ms = 10_000; // 10 seconds let (mut state, ctx, last_entry_id) = setup_single_voter_leader("/tmp/test_large_lease", lease_duration_ms).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Action: Single flush last_entry_id.store(1, Ordering::Relaxed); - state.handle_log_flushed(1, &ctx, &role_tx).await; + state.handle_log_flushed(1, &ctx, &internal_event_tx).await; assert!(state.is_lease_valid()); // Wait 1 second (much less than 10s lease) diff --git a/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs b/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs index d21962cf..a502088a 100644 --- a/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/membership_change_test.rs @@ -429,7 +429,7 @@ async fn test_handle_join_cluster_node_exists() { #[tokio::test(start_paused = true)] #[traced_test] async fn test_handle_join_cluster_quorum_failed() { - use crate::event::RaftEvent; + use crate::event::InboundEvent; let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -465,8 +465,8 @@ async fn test_handle_join_cluster_quorum_failed() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (sender, mut receiver) = >::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let (raft_tx, _raft_rx) = mpsc::channel::(1); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let (raft_tx, _raft_rx) = mpsc::channel::(1); // Fire-and-forget: returns Ok immediately, entry stored in pending_commit_actions let result = state @@ -479,7 +479,7 @@ async fn test_handle_join_cluster_quorum_failed() { }, sender, &context, - &role_tx, + &internal_event_tx, ) .await; @@ -497,7 +497,7 @@ async fn test_handle_join_cluster_quorum_failed() { tokio::time::advance(Duration::from_millis(200)).await; // tick() detects the expired NodeJoin and sends deadline_exceeded to the client - state.tick(&role_tx, &raft_tx, &context).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &context).await.unwrap(); // Client receives deadline_exceeded: join never committed (no quorum) let status = receiver.recv().await.unwrap().unwrap_err(); @@ -848,10 +848,10 @@ mod stale_learner_tests { }, ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Call handle_stale_learner (will timeout, but we captured the config) - let _ = leader.handle_stale_learner(101, &role_tx, &ctx).await; + let _ = leader.handle_stale_learner(101, &internal_event_tx, &ctx).await; // Validate: Correct BatchRemove config change was created let payloads = captured_payloads.lock(); @@ -898,7 +898,7 @@ mod check_learner_progress_tests { use crate::MockMembership; use crate::RaftContext; - use crate::event::RoleEvent; + use crate::event::InternalEvent; use crate::raft_role::leader_state::{LeaderState, PendingPromotion}; use crate::raft_role::role_state::RaftRoleState; use crate::test_utils::mock::MockTypeConfig; @@ -935,20 +935,26 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); // Force first check - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(95))]); // First call - should process - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 1); // Second call immediately - should be throttled - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 1); // No change // Verify only 1 event sent let mut event_count = 0; - while role_rx.try_recv().is_ok() { + while internal_event_rx.try_recv().is_ok() { event_count += 1; } assert_eq!(event_count, 1, "Should only send 1 event"); @@ -963,10 +969,13 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); // Force check - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(95))]); // gap = 100 - 95 = 5 - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 1); assert_eq!(leader.pending_promotions[0].node_id, 100); @@ -982,10 +991,13 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(94))]); // gap = 100 - 94 = 6 > 5 - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 0); } @@ -1000,10 +1012,13 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, None)]); // match_index = 0, gap = 100 - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 0); } @@ -1019,17 +1034,20 @@ mod check_learner_progress_tests { // Manually add learner 100 to pending queue leader.pending_promotions.push_back(PendingPromotion::new(100, Instant::now())); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(95))]); // caught_up - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); // Should still have only 1 entry assert_eq!(leader.pending_promotions.len(), 1); // Should not send event (all learners already pending) assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should not send event for duplicate" ); } @@ -1054,10 +1072,13 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(95))]); // caught_up - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!( leader.pending_promotions.len(), @@ -1074,10 +1095,13 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(95))]); // caught_up - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 1); } @@ -1089,13 +1113,19 @@ mod check_learner_progress_tests { let mut leader = LeaderState::::new(1, ctx.node_config.clone()); leader.update_commit_index(100).unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::new(); - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); assert_eq!(leader.pending_promotions.len(), 0); - assert!(role_rx.try_recv().is_err(), "Should not send event"); + assert!( + internal_event_rx.try_recv().is_err(), + "Should not send event" + ); } /// Test: Verify node not in membership is skipped @@ -1124,13 +1154,16 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([ (100, Some(95)), // valid (999, Some(95)), // not in membership ]); - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); // Should only add the valid learner assert_eq!(leader.pending_promotions.len(), 1); @@ -1145,10 +1178,13 @@ mod check_learner_progress_tests { leader.update_commit_index(5).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(10))]); // learner_match > leader_commit - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); // checked_sub returns None → caught_up = true assert_eq!(leader.pending_promotions.len(), 1); @@ -1162,10 +1198,13 @@ mod check_learner_progress_tests { leader.update_commit_index(100).unwrap(); leader.last_learner_check = Instant::now() - Duration::from_secs(2); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let learner_progress = HashMap::from([(100, Some(95)), (200, Some(96))]); - leader.check_learner_progress(&learner_progress, &ctx, &role_tx).await.unwrap(); + leader + .check_learner_progress(&learner_progress, &ctx, &internal_event_tx) + .await + .unwrap(); // Both learners should be in queue assert_eq!(leader.pending_promotions.len(), 2); @@ -1175,8 +1214,8 @@ mod check_learner_progress_tests { // Should receive exactly 1 event let mut event_count = 0; - while let Ok(event) = role_rx.try_recv() { - if matches!(event, RoleEvent::PromoteReadyLearners) { + while let Ok(event) = internal_event_rx.try_recv() { + if matches!(event, InternalEvent::PromoteReadyLearners) { event_count += 1; } } @@ -1206,7 +1245,7 @@ mod pending_promotion_tests { use crate::MockRaftLog; use crate::MockReplicationCore; use crate::RaftContext; - use crate::event::RoleEvent; + use crate::event::InternalEvent; use crate::raft_role::leader_state::{ LeaderState, PendingPromotion, calculate_safe_batch_size, }; @@ -1223,8 +1262,8 @@ mod pending_promotion_tests { struct TestFixture { leader_state: LeaderState, raft_context: RaftContext, - role_tx: mpsc::UnboundedSender, - role_rx: mpsc::UnboundedReceiver, + internal_event_tx: mpsc::UnboundedSender, + internal_event_rx: mpsc::UnboundedReceiver, captured_payloads: Arc>>, } @@ -1266,7 +1305,7 @@ mod pending_promotion_tests { ); raft_context.handlers.replication_handler = replication_handler; - let (role_tx, role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); let mut cfg = node_config(&format!("/tmp/{test_name}")); cfg.raft.batching.max_batch_size = 1; // Short timeout: quorum cannot complete in unit tests @@ -1277,8 +1316,8 @@ mod pending_promotion_tests { TestFixture { leader_state, raft_context, - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, captured_payloads, } } @@ -1346,7 +1385,7 @@ mod pending_promotion_tests { // Result is Err(Timeout): quorum cannot complete without a running Raft loop let _ = fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.internal_event_tx) .await; // Verify the correct BatchPromote config was submitted with all 3 nodes @@ -1424,10 +1463,10 @@ mod pending_promotion_tests { raft_context.node_config = Arc::new(node_config.clone()); let mut leader_state = LeaderState::new(1, Arc::new(node_config)); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Call handle_stale_learner (will timeout, but we captured the config) - let _ = leader_state.handle_stale_learner(1, &role_tx, &raft_context).await; + let _ = leader_state.handle_stale_learner(1, &internal_event_tx, &raft_context).await; // Validate: Correct BatchRemove config change was created let payloads = captured_payloads.lock(); @@ -1474,7 +1513,7 @@ mod pending_promotion_tests { let _ = fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.internal_event_tx) .await; // Verify FIFO: node 1 (older) was submitted, not node 2 @@ -1516,7 +1555,7 @@ mod pending_promotion_tests { let result = fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.internal_event_tx) .await; // Fire-and-forget: 9 submitted successfully, returns Ok @@ -1528,8 +1567,8 @@ mod pending_promotion_tests { // Step 6: exactly one PromoteReadyLearners event emitted for the 1 remaining item let reschedule_count = { let mut count = 0; - while let Ok(event) = fixture.role_rx.try_recv() { - if matches!(event, RoleEvent::PromoteReadyLearners) { + while let Ok(event) = fixture.internal_event_rx.try_recv() { + if matches!(event, InternalEvent::PromoteReadyLearners) { count += 1; } } @@ -1558,7 +1597,7 @@ mod pending_promotion_tests { (1..=2).map(|id| PendingPromotion::new(id, Instant::now())).collect(); let result = fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.internal_event_tx) .await; // Fire-and-forget: 1 submitted, returns Ok @@ -1582,7 +1621,7 @@ mod pending_promotion_tests { fixture.leader_state.update_current_term(2); let result = fixture .leader_state - .handle_promote_ready_learners(&fixture.raft_context, &fixture.role_tx) + .handle_promote_ready_learners(&fixture.raft_context, &fixture.internal_event_tx) .await; // Fire-and-forget: term change does not prevent submission @@ -1694,9 +1733,9 @@ mod zombie_purge_tests { let node_config = raft_context.node_config(); let mut leader = LeaderState::::new(1, node_config); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - let result = leader.handle_zombie_node(5, &role_tx, &raft_context).await; + let result = leader.handle_zombie_node(5, &internal_event_tx, &raft_context).await; assert!(result.is_ok(), "handle_zombie_node must not fail"); } @@ -1733,9 +1772,9 @@ mod zombie_purge_tests { let node_config = raft_context.node_config(); let mut leader = LeaderState::::new(1, node_config); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - let result = leader.handle_zombie_node(5, &role_tx, &raft_context).await; + let result = leader.handle_zombie_node(5, &internal_event_tx, &raft_context).await; assert!(result.is_ok(), "handle_zombie_node must not fail"); } @@ -1778,9 +1817,9 @@ mod zombie_purge_tests { let node_config = raft_context.node_config(); let mut leader = LeaderState::::new(1, node_config); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - let result = leader.handle_zombie_node(5, &role_tx, &raft_context).await; + let result = leader.handle_zombie_node(5, &internal_event_tx, &raft_context).await; assert!(result.is_ok()); assert!( captured_payloads.lock().is_empty(), diff --git a/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs b/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs index 8a248add..dc41a434 100644 --- a/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs @@ -1,13 +1,13 @@ //! TDD tests for `pending_commit_actions` unified post-commit queue (#333). //! //! Red phase: these tests drive the implementation of: -//! - `LeaderState::drain_commit_actions(new_commit_index, role_tx)` +//! - `LeaderState::drain_commit_actions(new_commit_index, internal_event_tx)` //! - tick() step 5: deadline scan for expired `pending_commit_actions` //! //! Three key behaviors: -//! 1. `drain_commit_actions` fires `RoleEvent::NoopCommitted` when a `LeaderNoop` entry commits. -//! 2. `drain_commit_actions` fires `RoleEvent::JoinCommitted` when a `NodeJoin` entry commits. -//! 3. tick() sends `RoleEvent::BecomeFollower` for expired noop entries (leadership lost). +//! 1. `drain_commit_actions` fires `InternalEvent::NoopCommitted` when a `LeaderNoop` entry commits. +//! 2. `drain_commit_actions` fires `InternalEvent::JoinCommitted` when a `NodeJoin` entry commits. +//! 3. tick() sends `InternalEvent::BecomeFollower` for expired noop entries (leadership lost). use std::time::Duration; @@ -15,7 +15,7 @@ use tokio::sync::{mpsc, watch}; use tokio::time::Instant; use crate::MaybeCloneOneshot; -use crate::event::RoleEvent; +use crate::event::InternalEvent; use crate::maybe_clone_oneshot::RaftOneshot; use crate::raft_role::leader_state::{LeaderState, PostCommitAction, PostCommitEntry}; use crate::raft_role::role_state::RaftRoleState; @@ -27,15 +27,15 @@ use crate::test_utils::mock::MockTypeConfig; async fn setup() -> ( LeaderState, crate::RaftContext, - mpsc::UnboundedSender, - mpsc::Sender, + mpsc::UnboundedSender, + mpsc::Sender, ) { let (_shutdown_tx, shutdown_rx) = watch::channel(()); let ctx = MockBuilder::new(shutdown_rx).build_context(); let state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (raft_tx, _raft_rx) = mpsc::channel(1); - (state, ctx, role_tx, raft_tx) + (state, ctx, internal_event_tx, raft_tx) } fn noop_entry( @@ -79,19 +79,19 @@ fn expired() -> Instant { // ── Test 1: drain_commit_actions fires NoopCommitted ───────────────────────── -/// `drain_commit_actions` fires `RoleEvent::NoopCommitted` when commit_index +/// `drain_commit_actions` fires `InternalEvent::NoopCommitted` when commit_index /// reaches the noop entry's log index. #[tokio::test] async fn test_drain_commit_actions_fires_noop_committed() { - let (mut state, ctx, _role_tx, _raft_tx) = setup().await; + let (mut state, ctx, _internal_event_tx, _raft_tx) = setup().await; // Insert noop at log index 5, term 2. state.pending_commit_actions.insert(5, noop_entry(2, far_future())); // Advance commit to index 5 — should drain and fire NoopCommitted. - let mut role_rx = { - let (tx, rx) = mpsc::unbounded_channel::(); - // Replace role_tx with one we can read. + let mut internal_event_rx = { + let (tx, rx) = mpsc::unbounded_channel::(); + // Replace internal_event_tx with one we can read. state.drain_commit_actions(5, &ctx, &tx).await; rx }; @@ -109,10 +109,10 @@ async fn test_drain_commit_actions_fires_noop_committed() { "noop_log_id must be set synchronously by drain_commit_actions" ); - // RoleEvent::NoopCommitted { term: 2 } must have been sent (for notify_leader_change). - let event = role_rx.try_recv().expect("expected NoopCommitted event"); + // InternalEvent::NoopCommitted { term: 2 } must have been sent (for notify_leader_change). + let event = internal_event_rx.try_recv().expect("expected NoopCommitted event"); match event { - RoleEvent::NoopCommitted { term } => assert_eq!(term, 2), + InternalEvent::NoopCommitted { term } => assert_eq!(term, 2), other => panic!("expected NoopCommitted, got {:?}", other), } } @@ -123,12 +123,12 @@ async fn test_drain_commit_actions_fires_noop_committed() { /// reaches the NodeJoin entry's log index — no channel roundtrip needed. #[tokio::test] async fn test_drain_commit_actions_sends_join_response_directly() { - let (mut state, ctx, _role_tx, _raft_tx) = setup().await; + let (mut state, ctx, _internal_event_tx, _raft_tx) = setup().await; let (entry, mut join_rx) = join_entry(42, far_future()); state.pending_commit_actions.insert(7, entry); - let (tx, mut role_rx) = mpsc::unbounded_channel::(); + let (tx, mut internal_event_rx) = mpsc::unbounded_channel::(); state.drain_commit_actions(7, &ctx, &tx).await; // Entry must be drained. @@ -139,7 +139,7 @@ async fn test_drain_commit_actions_sends_join_response_directly() { // No JoinCommitted event — response is delivered directly to join_rx. assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no JoinCommitted event should be sent — join response goes directly to client" ); @@ -155,11 +155,11 @@ async fn test_drain_commit_actions_sends_join_response_directly() { /// `drain_commit_actions` must not drain an entry whose index is after commit_index. #[tokio::test] async fn test_drain_commit_actions_skips_uncommitted_entries() { - let (mut state, ctx, _role_tx, _raft_tx) = setup().await; + let (mut state, ctx, _internal_event_tx, _raft_tx) = setup().await; state.pending_commit_actions.insert(10, noop_entry(3, far_future())); - let (tx, mut role_rx) = mpsc::unbounded_channel::(); + let (tx, mut internal_event_rx) = mpsc::unbounded_channel::(); state.drain_commit_actions(9, &ctx, &tx).await; // commit only up to 9 assert_eq!( @@ -168,7 +168,7 @@ async fn test_drain_commit_actions_skips_uncommitted_entries() { "uncommitted entry must remain" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no event should be sent for uncommitted entry" ); } @@ -177,18 +177,18 @@ async fn test_drain_commit_actions_skips_uncommitted_entries() { /// When a `LeaderNoop` entry has expired (deadline passed), tick() must: /// - Remove the entry from `pending_commit_actions` -/// - Send `RoleEvent::BecomeFollower(None)` (lost quorum, step down) +/// - Send `InternalEvent::BecomeFollower(None)` (lost quorum, step down) #[tokio::test] async fn test_tick_drains_expired_noop_sends_become_follower() { let (_shutdown_tx, shutdown_rx) = watch::channel(()); let ctx = MockBuilder::new(shutdown_rx).build_context(); let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); let (raft_tx, _raft_rx) = mpsc::channel(1); state.pending_commit_actions.insert(5, noop_entry(2, expired())); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); // Entry must be removed. assert!( @@ -197,9 +197,9 @@ async fn test_tick_drains_expired_noop_sends_become_follower() { ); // Must have received BecomeFollower(None). - let event = role_rx.try_recv().expect("expected BecomeFollower event"); + let event = internal_event_rx.try_recv().expect("expected BecomeFollower event"); match event { - RoleEvent::BecomeFollower(None) => {} + InternalEvent::BecomeFollower(None) => {} other => panic!("expected BecomeFollower(None), got {:?}", other), } } @@ -212,12 +212,12 @@ async fn test_tick_keeps_live_noop_in_commit_actions() { let (_shutdown_tx, shutdown_rx) = watch::channel(()); let ctx = MockBuilder::new(shutdown_rx).build_context(); let mut state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); let (raft_tx, _raft_rx) = mpsc::channel(1); state.pending_commit_actions.insert(5, noop_entry(2, far_future())); - state.tick(&role_tx, &raft_tx, &ctx).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &ctx).await.unwrap(); assert_eq!( state.pending_commit_actions.len(), @@ -225,5 +225,8 @@ async fn test_tick_keeps_live_noop_in_commit_actions() { "live noop entry must not be removed by tick" ); // No events should be sent for live entry. - assert!(role_rx.try_recv().is_err(), "no event for live noop entry"); + assert!( + internal_event_rx.try_recv().is_err(), + "no event for live noop entry" + ); } diff --git a/d-engine-core/src/raft_role/leader_state_test/pending_lease_reads_test.rs b/d-engine-core/src/raft_role/leader_state_test/pending_lease_reads_test.rs index ba7e96dd..329ec2eb 100644 --- a/d-engine-core/src/raft_role/leader_state_test/pending_lease_reads_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/pending_lease_reads_test.rs @@ -83,8 +83,8 @@ async fn setup_multi_voter_expired_lease( ) -> ( LeaderState, crate::raft_context::RaftContext, - mpsc::UnboundedSender, - mpsc::UnboundedReceiver, + mpsc::UnboundedSender, + mpsc::UnboundedReceiver, ) { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -147,8 +147,8 @@ async fn setup_multi_voter_expired_lease( state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); // lease is expired by default (timestamp = 0) - let (role_tx, role_rx) = mpsc::unbounded_channel(); - (state, context, role_tx, role_rx) + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); + (state, context, internal_event_tx, internal_event_rx) } // ============================================================================ @@ -161,10 +161,11 @@ async fn setup_multi_voter_expired_lease( #[tokio::test] #[traced_test] async fn test_lease_read_expired_pushes_to_pending_lease_reads() { - let (mut state, context, role_tx, _role_rx) = setup_multi_voter_expired_lease( - "/tmp/test_lease_read_expired_pushes_to_pending_lease_reads", - ) - .await; + let (mut state, context, internal_event_tx, _internal_event_rx) = + setup_multi_voter_expired_lease( + "/tmp/test_lease_read_expired_pushes_to_pending_lease_reads", + ) + .await; assert!(!state.is_lease_valid(), "precondition: lease expired"); @@ -174,7 +175,7 @@ async fn test_lease_read_expired_pushes_to_pending_lease_reads() { // flush_cmd_buffers must return quickly (must not block awaiting quorum) let flush_result = tokio::time::timeout( Duration::from_millis(200), - state.flush_cmd_buffers(&context, &role_tx), + state.flush_cmd_buffers(&context, &internal_event_tx), ) .await; @@ -201,7 +202,7 @@ async fn test_lease_read_expired_pushes_to_pending_lease_reads() { #[tokio::test] #[traced_test] async fn test_pending_lease_reads_drained_on_quorum_ack() { - let (mut state, mut context, role_tx, _role_rx) = + let (mut state, mut context, internal_event_tx, _internal_event_rx) = setup_multi_voter_expired_lease("/tmp/test_pending_lease_reads_drained_on_quorum_ack") .await; @@ -217,7 +218,7 @@ async fn test_pending_lease_reads_drained_on_quorum_ack() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(make_lease_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.unwrap(); assert_eq!( state.pending_lease_reads.len(), @@ -228,7 +229,7 @@ async fn test_pending_lease_reads_drained_on_quorum_ack() { // Simulate quorum ACK from peer 2 with match_index=10. // Majority = 2/3: leader(self) + peer 2 is sufficient. state - .handle_append_result(2, Ok(quorum_ack(1, 10)), &context, &role_tx) + .handle_append_result(2, Ok(quorum_ack(1, 10)), &context, &internal_event_tx) .await .unwrap(); @@ -292,9 +293,9 @@ async fn test_single_voter_lease_read_served_immediately_on_expired_lease() { assert!(!state.is_lease_valid(), "precondition: lease expired"); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state.push_client_cmd(make_lease_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.unwrap(); // Must not be queued — single-voter serves immediately assert_eq!( @@ -336,9 +337,9 @@ async fn test_pending_lease_reads_timeout_on_tick() { deadline: Instant::now() - Duration::from_secs(1), }); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (raft_tx, _raft_rx) = mpsc::channel(1); - state.tick(&role_tx, &raft_tx, &context).await.unwrap(); + state.tick(&internal_event_tx, &raft_tx, &context).await.unwrap(); // Queue must be cleared assert_eq!( @@ -407,10 +408,11 @@ async fn test_pending_lease_reads_cleared_on_stepdown() { #[tokio::test] #[traced_test] async fn test_multiple_lease_reads_batched_on_single_quorum_ack() { - let (mut state, mut context, role_tx, _role_rx) = setup_multi_voter_expired_lease( - "/tmp/test_multiple_lease_reads_batched_on_single_quorum_ack", - ) - .await; + let (mut state, mut context, internal_event_tx, _internal_event_rx) = + setup_multi_voter_expired_lease( + "/tmp/test_multiple_lease_reads_batched_on_single_quorum_ack", + ) + .await; context.handlers.replication_handler.expect_handle_success_response().returning( |_, _, _, _| { @@ -427,7 +429,7 @@ async fn test_multiple_lease_reads_batched_on_single_quorum_ack() { for _ in 0..3 { let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(make_lease_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.unwrap(); receivers.push(resp_rx); } @@ -439,7 +441,7 @@ async fn test_multiple_lease_reads_batched_on_single_quorum_ack() { // Single quorum ACK must drain all 3 state - .handle_append_result(2, Ok(quorum_ack(1, 10)), &context, &role_tx) + .handle_append_result(2, Ok(quorum_ack(1, 10)), &context, &internal_event_tx) .await .unwrap(); @@ -478,13 +480,14 @@ async fn test_multiple_lease_reads_batched_on_single_quorum_ack() { #[tokio::test] #[traced_test] async fn test_pending_lease_reads_drained_on_higher_term_stepdown() { - use crate::event::RoleEvent; + use crate::event::InternalEvent; use d_engine_proto::server::replication::AppendEntriesResponse; - let (mut state, mut context, role_tx, mut role_rx) = setup_multi_voter_expired_lease( - "/tmp/test_pending_lease_reads_drained_on_higher_term_stepdown", - ) - .await; + let (mut state, mut context, internal_event_tx, mut internal_event_rx) = + setup_multi_voter_expired_lease( + "/tmp/test_pending_lease_reads_drained_on_higher_term_stepdown", + ) + .await; context.handlers.replication_handler.expect_handle_success_response().returning( |_, _, _, _| { @@ -499,7 +502,7 @@ async fn test_pending_lease_reads_drained_on_higher_term_stepdown() { // Queue one lease read while lease is expired let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(make_lease_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.unwrap(); + state.flush_cmd_buffers(&context, &internal_event_tx).await.unwrap(); assert_eq!( state.pending_lease_reads.len(), 1, @@ -512,14 +515,19 @@ async fn test_pending_lease_reads_drained_on_higher_term_stepdown() { term: 10, // higher than leader term=1 result: None, }; - let result = state.handle_append_result(2, Ok(higher_term_resp), &context, &role_tx).await; + let result = state + .handle_append_result(2, Ok(higher_term_resp), &context, &internal_event_tx) + .await; // handle_append_result must return Err(HigherTerm) assert!(result.is_err(), "higher term must return Err"); // BecomeFollower event must be sent to the Raft loop assert!( - matches!(role_rx.try_recv(), Ok(RoleEvent::BecomeFollower(_))), + matches!( + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) + ), "BecomeFollower event must be queued" ); diff --git a/d-engine-core/src/raft_role/leader_state_test/pending_reads_test.rs b/d-engine-core/src/raft_role/leader_state_test/pending_reads_test.rs index 159c0b2d..689f6e45 100644 --- a/d-engine-core/src/raft_role/leader_state_test/pending_reads_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/pending_reads_test.rs @@ -48,7 +48,7 @@ use crate::PeerUpdate; use crate::RaftNodeConfig; use crate::ReadConsistencyPolicy; use crate::convert::safe_kv_bytes; -use crate::event::RaftEvent; +use crate::event::InboundEvent; use crate::maybe_clone_oneshot::{MaybeCloneOneshot, RaftOneshot}; use crate::raft_role::leader_state::{LeaderState, PendingReadBatch}; use crate::raft_role::role_state::RaftRoleState; @@ -107,8 +107,8 @@ fn sm_with_last_applied(index: u64) -> MockStateMachine { type MultiVoterFixture = ( LeaderState, crate::raft_context::RaftContext, - mpsc::UnboundedSender, - mpsc::UnboundedReceiver, + mpsc::UnboundedSender, + mpsc::UnboundedReceiver, ); /// Set up a 3-voter cluster (leader=1, peers=2,3) ready for LinearizableRead tests. @@ -152,8 +152,8 @@ async fn setup_multi_voter( membership.expect_replication_peers().returning(move || peers.clone()); state.init_cluster_metadata(&Arc::new(membership)).await.unwrap(); - let (role_tx, role_rx) = mpsc::unbounded_channel(); - (state, context, role_tx, role_rx) + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); + (state, context, internal_event_tx, internal_event_rx) } /// Build a `MockRaftLog` that returns `Some(match_index)` for every @@ -208,7 +208,7 @@ async fn test_multi_voter_fast_path_linear_read_is_queued() { .times(1) .returning(|_, _, _, _, _| Ok(crate::PrepareResult::default())); - let (mut state, context, role_tx, _role_rx) = setup_multi_voter( + let (mut state, context, internal_event_tx, _internal_event_rx) = setup_multi_voter( "/tmp/test_multi_voter_fast_path_linear_read_is_queued", replication, raft_log_no_quorum(), @@ -222,7 +222,10 @@ async fn test_multi_voter_fast_path_linear_read_is_queued() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(linear_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("flush must succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("flush must succeed"); // After the fix: read is queued, not served. assert_eq!( @@ -272,7 +275,7 @@ async fn test_pending_reads_drained_by_quorum_ack() { }); // quorum_confirmed = calculate_majority_matched_index(...).is_some() - let (mut state, context, role_tx, _role_rx) = setup_multi_voter( + let (mut state, context, internal_event_tx, _internal_event_rx) = setup_multi_voter( "/tmp/test_pending_reads_drained_by_quorum_ack", replication, raft_log_with_quorum(0), @@ -303,7 +306,7 @@ async fn test_pending_reads_drained_by_quorum_ack() { // Simulate quorum ACK from peer 2. // Majority = 2/3: self + peer 2 is sufficient. state - .handle_append_result(2, Ok(quorum_ack(1, 0)), &context, &role_tx) + .handle_append_result(2, Ok(quorum_ack(1, 0)), &context, &internal_event_tx) .await .expect("handle_append_result must succeed"); @@ -371,11 +374,11 @@ async fn test_pending_reads_slow_path_drained_by_apply_completed() { ); assert_eq!(state.pending_reads.len(), 1, "precondition: read is queued"); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // SM applies entries up to index 5 — this is the commit chain completing. state - .handle_apply_completed(5, vec![], &context, &role_tx) + .handle_apply_completed(5, vec![], &context, &internal_event_tx) .await .expect("handle_apply_completed must succeed"); @@ -442,10 +445,13 @@ async fn test_pending_reads_partition_scenario_times_out() { }, ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let (raft_tx, _raft_rx) = mpsc::channel::(1); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let (raft_tx, _raft_rx) = mpsc::channel::(1); - state.tick(&role_tx, &raft_tx, &context).await.expect("tick must succeed"); + state + .tick(&internal_event_tx, &raft_tx, &context) + .await + .expect("tick must succeed"); // tick() must remove the expired entry. assert_eq!( @@ -568,7 +574,7 @@ async fn test_linearizable_read_served_immediately_with_valid_lease_in_multi_vot .times(1) .returning(|_, _, _, _, _| Ok(crate::PrepareResult::default())); - let (mut state, context, role_tx, _role_rx) = setup_multi_voter( + let (mut state, context, internal_event_tx, _internal_event_rx) = setup_multi_voter( "/tmp/test_linearizable_read_served_immediately_with_valid_lease_in_multi_voter", replication, raft_log_no_quorum(), @@ -584,7 +590,10 @@ async fn test_linearizable_read_served_immediately_with_valid_lease_in_multi_vot let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(linear_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("flush must succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("flush must succeed"); // Fast path taken: read must NOT be queued. assert_eq!( @@ -637,7 +646,7 @@ async fn test_linearizable_read_queued_when_sm_behind_despite_valid_lease() { .times(1) .returning(|_, _, _, _, _| Ok(crate::PrepareResult::default())); - let (mut state, context, role_tx, _role_rx) = setup_multi_voter( + let (mut state, context, internal_event_tx, _internal_event_rx) = setup_multi_voter( "/tmp/test_linearizable_read_queued_when_sm_behind_despite_valid_lease", replication, raft_log_no_quorum(), @@ -653,7 +662,10 @@ async fn test_linearizable_read_queued_when_sm_behind_despite_valid_lease() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(linear_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("flush must succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("flush must succeed"); // SM not caught up: read must be queued regardless of lease validity. assert_eq!( @@ -704,7 +716,7 @@ async fn test_linearizable_read_not_served_when_lease_expired_in_multi_voter() { .times(1) .returning(|_, _, _, _, _| Ok(crate::PrepareResult::default())); - let (mut state, context, role_tx, _role_rx) = setup_multi_voter( + let (mut state, context, internal_event_tx, _internal_event_rx) = setup_multi_voter( "/tmp/test_linearizable_read_not_served_when_lease_expired_in_multi_voter", replication, raft_log_no_quorum(), @@ -720,7 +732,10 @@ async fn test_linearizable_read_not_served_when_lease_expired_in_multi_voter() { let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); state.push_client_cmd(linear_read_cmd(resp_tx), &context); - state.flush_cmd_buffers(&context, &role_tx).await.expect("flush must succeed"); + state + .flush_cmd_buffers(&context, &internal_event_tx) + .await + .expect("flush must succeed"); // Expired lease → slow path: read must be queued, not served inline. assert_eq!( diff --git a/d-engine-core/src/raft_role/leader_state_test/replication_test.rs b/d-engine-core/src/raft_role/leader_state_test/replication_test.rs index 6dc2bfd1..cedc293f 100644 --- a/d-engine-core/src/raft_role/leader_state_test/replication_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/replication_test.rs @@ -11,7 +11,7 @@ use crate::MockReplicationCore; use crate::RaftContext; use crate::RaftRequestWithSignal; use crate::client::ClientResponse; -use crate::event::RoleEvent; +use crate::event::InternalEvent; use crate::maybe_clone_oneshot::RaftOneshot; use crate::raft_role::leader_state::LeaderState; use crate::role_state::RaftRoleState; @@ -192,8 +192,11 @@ async fn test_process_batch_quorum_achieved() { let batch = VecDeque::from(vec![mock_request(tx1), mock_request(tx2)]); let receivers = vec![rx1, rx2]; - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let result = context.state.process_batch(batch, &role_tx, &context.raft_context).await; + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let result = context + .state + .process_batch(batch, &internal_event_tx, &context.raft_context) + .await; assert!(result.is_ok()); @@ -201,12 +204,15 @@ async fn test_process_batch_quorum_achieved() { // MemFirst: set last_entry_id=6 before flush. last_entry_id.store(6, Ordering::Relaxed); context.state.cluster_metadata.single_voter = true; - context.state.handle_log_flushed(6, &context.raft_context, &role_tx).await; + context + .state + .handle_log_flushed(6, &context.raft_context, &internal_event_tx) + .await; assert_eq!(context.state.shared_state().commit_index, 6); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); // Verify client responses @@ -372,10 +378,13 @@ async fn test_process_batch_higher_term() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (tx1, mut rx1) = >::new(); let batch = VecDeque::from(vec![mock_request(tx1)]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // process_batch queues the write and returns Ok - let result = context.state.process_batch(batch, &role_tx, &context.raft_context).await; + let result = context + .state + .process_batch(batch, &internal_event_tx, &context.raft_context) + .await; assert!(result.is_ok()); // Trigger step-down via higher-term peer response @@ -386,14 +395,19 @@ async fn test_process_batch_higher_term() { }; let ha_result = context .state - .handle_append_result(2, Ok(higher_term_resp), &context.raft_context, &role_tx) + .handle_append_result( + 2, + Ok(higher_term_resp), + &context.raft_context, + &internal_event_tx, + ) .await; assert!(ha_result.is_err()); assert_eq!(context.state.current_term(), 10); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::BecomeFollower(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) )); // Pending write receives TermOutdated error @@ -684,15 +698,21 @@ async fn test_single_node_cluster_commit_index() { let (tx1, rx1) = >::new(); let batch = VecDeque::from(vec![mock_request(tx1)]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = context.state.process_batch(batch, &role_tx, &context.raft_context).await; + let result = context + .state + .process_batch(batch, &internal_event_tx, &context.raft_context) + .await; assert!(result.is_ok()); // Simulate the async LogFlushed event that BufferedRaftLog's batch_processor fires // after fsync. MemFirst: set last_entry_id=7 before flush. last_entry_id.store(7, Ordering::Relaxed); - context.state.handle_log_flushed(7, &context.raft_context, &role_tx).await; + context + .state + .handle_log_flushed(7, &context.raft_context, &internal_event_tx) + .await; assert_eq!( context.state.shared_state().commit_index, @@ -700,8 +720,8 @@ async fn test_single_node_cluster_commit_index() { "Single-node: commit_index should advance to last_entry_id after LogFlushed" ); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); let mut rx = rx1; @@ -750,9 +770,12 @@ async fn test_multi_node_cluster_empty_peer_updates_commit_index() { use crate::maybe_clone_oneshot::MaybeCloneOneshot; let (tx1, rx1) = >::new(); let batch = VecDeque::from(vec![mock_request(tx1)]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = context.state.process_batch(batch, &role_tx, &context.raft_context).await; + let result = context + .state + .process_batch(batch, &internal_event_tx, &context.raft_context) + .await; assert!(result.is_ok()); assert_eq!( @@ -761,7 +784,7 @@ async fn test_multi_node_cluster_empty_peer_updates_commit_index() { "Multi-node: commit does not advance without peer ACKs" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No NotifyNewCommitIndex without commit advancement" ); @@ -830,9 +853,12 @@ async fn test_multi_node_cluster_with_peer_updates_commit_index() { let (tx1, rx1) = >::new(); let (tx2, rx2) = >::new(); let batch = VecDeque::from(vec![mock_request(tx1), mock_request(tx2)]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = context.state.process_batch(batch, &role_tx, &context.raft_context).await; + let result = context + .state + .process_batch(batch, &internal_event_tx, &context.raft_context) + .await; assert!(result.is_ok()); // Simulate peer 2 responding — triggers calculate_majority_matched_index → commit=6 @@ -842,15 +868,15 @@ async fn test_multi_node_cluster_with_peer_updates_commit_index() { 2, Ok(success_response(1, 6)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); assert_eq!(context.state.shared_state().commit_index, 6); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); let mut rx = rx1; @@ -932,17 +958,20 @@ async fn test_verify_internal_quorum_success() { wait_for_apply_event: false, }; - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.execute_request_immediately(req, &raft_context, &role_tx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state + .execute_request_immediately(req, &raft_context, &internal_event_tx) + .await + .unwrap(); // Simulate the async LogFlushed event. MemFirst: set last_entry_id=5 before flush. last_entry_id.store(5, Ordering::Relaxed); - state.handle_log_flushed(5, &raft_context, &role_tx).await; + state.handle_log_flushed(5, &raft_context, &internal_event_tx).await; assert_eq!(state.shared_state().commit_index, 5); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); // Response received: write_success @@ -1000,8 +1029,11 @@ async fn test_verify_internal_quorum_verifiable_failure() { wait_for_apply_event: false, }; - let (role_tx, _) = mpsc::unbounded_channel(); - state.execute_request_immediately(req, &raft_context, &role_tx).await.unwrap(); + let (internal_event_tx, _) = mpsc::unbounded_channel(); + state + .execute_request_immediately(req, &raft_context, &internal_event_tx) + .await + .unwrap(); // Write stays pending — no commit advancement assert!( @@ -1089,8 +1121,11 @@ async fn test_verify_internal_quorum_non_verifiable_failure() { wait_for_apply_event: false, }; - let (role_tx, _) = mpsc::unbounded_channel(); - state.execute_request_immediately(req, &raft_context, &role_tx).await.unwrap(); + let (internal_event_tx, _) = mpsc::unbounded_channel(); + state + .execute_request_immediately(req, &raft_context, &internal_event_tx) + .await + .unwrap(); // Write is pending — 4-node cluster, no peer ACKs received assert!( @@ -1147,8 +1182,11 @@ async fn test_verify_internal_quorum_partial_timeouts() { wait_for_apply_event: false, }; - let (role_tx, _) = mpsc::unbounded_channel(); - state.execute_request_immediately(req, &raft_context, &role_tx).await.unwrap(); + let (internal_event_tx, _) = mpsc::unbounded_channel(); + state + .execute_request_immediately(req, &raft_context, &internal_event_tx) + .await + .unwrap(); // Write is pending — partial peer timeouts, no commit advancement assert!( @@ -1202,8 +1240,11 @@ async fn test_verify_internal_quorum_all_timeouts() { wait_for_apply_event: false, }; - let (role_tx, _) = mpsc::unbounded_channel(); - state.execute_request_immediately(req, &raft_context, &role_tx).await.unwrap(); + let (internal_event_tx, _) = mpsc::unbounded_channel(); + state + .execute_request_immediately(req, &raft_context, &internal_event_tx) + .await + .unwrap(); // Write is pending — all peers timed out assert!( @@ -1264,8 +1305,11 @@ async fn test_verify_internal_quorum_higher_term() { wait_for_apply_event: false, }; - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.execute_request_immediately(req, &raft_context, &role_tx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state + .execute_request_immediately(req, &raft_context, &internal_event_tx) + .await + .unwrap(); // Trigger step-down via higher-term peer response let higher_term_resp = AppendEntriesResponse { @@ -1274,13 +1318,13 @@ async fn test_verify_internal_quorum_higher_term() { result: None, }; let result = state - .handle_append_result(2, Ok(higher_term_resp), &raft_context, &role_tx) + .handle_append_result(2, Ok(higher_term_resp), &raft_context, &internal_event_tx) .await; assert!(result.is_err()); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::BecomeFollower(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::BecomeFollower(_)) )); // Pending write receives TermOutdated error @@ -1336,8 +1380,8 @@ async fn test_verify_internal_quorum_critical_failure() { wait_for_apply_event: false, }; - let (role_tx, _) = mpsc::unbounded_channel(); - let result = state.execute_request_immediately(req, &raft_context, &role_tx).await; + let (internal_event_tx, _) = mpsc::unbounded_channel(); + let result = state.execute_request_immediately(req, &raft_context, &internal_event_tx).await; assert!(result.is_err()); assert!(matches!( @@ -1416,8 +1460,11 @@ async fn test_execute_and_process_raft_rpc_multi_node_empty_peer_updates() { "clock failed to advance past 0 within 50ms" ); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - let result = context.state.process_batch(batch, &role_tx, &context.raft_context).await; + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + let result = context + .state + .process_batch(batch, &internal_event_tx, &context.raft_context) + .await; assert!(result.is_ok()); assert_eq!( @@ -1426,7 +1473,7 @@ async fn test_execute_and_process_raft_rpc_multi_node_empty_peer_updates() { "Multi-node cluster: commit stays at 5 without peer ACKs (not last_entry_id=10)" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No NotifyNewCommitIndex without commit advancement" ); @@ -1566,13 +1613,13 @@ async fn test_handle_append_result_two_node_quorum_achieved() { state.init_cluster_metadata(&raft_context.membership).await.unwrap(); // initial commit_index = 0 - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); state .handle_append_result( peer2_id, Ok(success_response(1, 3)), &raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1583,8 +1630,8 @@ async fn test_handle_append_result_two_node_quorum_achieved() { "two-node: commit advances to 3 (quorum: leader + peer2 = 2/2)" ); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); } @@ -1633,7 +1680,7 @@ async fn test_handle_append_result_three_node_quorum_all_peers() { ); context.raft_context.storage.raft_log = Arc::new(raft_log); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Peer 2 responds → quorum (2/3), commit advances context @@ -1642,7 +1689,7 @@ async fn test_handle_append_result_three_node_quorum_all_peers() { 2, Ok(success_response(1, 10)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1653,8 +1700,8 @@ async fn test_handle_append_result_three_node_quorum_all_peers() { "commit advances after peer 2" ); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); // Peer 3 responds → already committed, calculate_majority returns None @@ -1664,7 +1711,7 @@ async fn test_handle_append_result_three_node_quorum_all_peers() { 3, Ok(success_response(1, 10)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1715,7 +1762,7 @@ async fn test_handle_append_result_three_node_quorum_partial_timeout() { raft_log.expect_calculate_majority_matched_index().returning(|_, _, _| Some(10)); context.raft_context.storage.raft_log = Arc::new(raft_log); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // Only peer 2 responds; peer 3 never does context @@ -1724,7 +1771,7 @@ async fn test_handle_append_result_three_node_quorum_partial_timeout() { 2, Ok(success_response(1, 10)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1735,8 +1782,8 @@ async fn test_handle_append_result_three_node_quorum_partial_timeout() { "quorum achieved with partial timeout (2/3 nodes = majority)" ); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); } @@ -1801,7 +1848,7 @@ async fn test_handle_append_result_five_node_quorum_majority() { ); context.raft_context.storage.raft_log = Arc::new(raft_log); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // 3 peers respond → majority of 5 (need ≥3) achieved context @@ -1810,7 +1857,7 @@ async fn test_handle_append_result_five_node_quorum_majority() { 2, Ok(success_response(1, 10)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1821,8 +1868,8 @@ async fn test_handle_append_result_five_node_quorum_majority() { "commit advances (4/5 ≥ majority)" ); assert!(matches!( - role_rx.try_recv(), - Ok(RoleEvent::NotifyNewCommitIndex(_)) + internal_event_rx.try_recv(), + Ok(InternalEvent::NotifyNewCommitIndex(_)) )); // Remaining peers just confirm — no second commit advance @@ -1833,7 +1880,7 @@ async fn test_handle_append_result_five_node_quorum_majority() { peer_id, Ok(success_response(1, 10)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1864,7 +1911,7 @@ async fn test_handle_append_result_three_node_no_quorum_all_timeouts() { // No handle_success_response or calculate_majority_matched_index expected (early return on Err) - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); for peer_id in [2u32, 3u32] { let result = context @@ -1873,7 +1920,7 @@ async fn test_handle_append_result_three_node_no_quorum_all_timeouts() { peer_id, Err(crate::Error::Fatal("simulated timeout".to_string())), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await; assert!(result.is_ok(), "RPC failure is tolerated, not propagated"); @@ -1885,7 +1932,7 @@ async fn test_handle_append_result_three_node_no_quorum_all_timeouts() { "commit unchanged — all peers timed out, no quorum" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no NotifyNewCommitIndex without commit advancement" ); } @@ -1929,7 +1976,7 @@ async fn test_handle_append_result_three_node_no_quorum_single_peer_insufficient raft_log.expect_calculate_majority_matched_index().returning(|_, _, _| None); context.raft_context.storage.raft_log = Arc::new(raft_log); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); context .state @@ -1937,7 +1984,7 @@ async fn test_handle_append_result_three_node_no_quorum_single_peer_insufficient 2, Ok(success_response(1, 4)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -1948,7 +1995,7 @@ async fn test_handle_append_result_three_node_no_quorum_single_peer_insufficient "commit unchanged — calculate_majority returned None" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no NotifyNewCommitIndex without commit advancement" ); } @@ -2009,7 +2056,7 @@ async fn test_handle_append_result_five_node_no_quorum_minority() { raft_log.expect_calculate_majority_matched_index().returning(|_, _, _| None); context.raft_context.storage.raft_log = Arc::new(raft_log); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); context .state @@ -2017,7 +2064,7 @@ async fn test_handle_append_result_five_node_no_quorum_minority() { 2, Ok(success_response(1, 10)), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await .unwrap(); @@ -2028,7 +2075,7 @@ async fn test_handle_append_result_five_node_no_quorum_minority() { "commit unchanged — minority response (2/5 < majority)" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no NotifyNewCommitIndex without quorum" ); } @@ -2054,7 +2101,7 @@ async fn test_handle_append_result_stale_term_ignored() { // No mock expectations — stale response triggers early return before any handler call - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); // term=0 is stale (leader_term=1), response ignored let stale_response = AppendEntriesResponse { @@ -2066,7 +2113,12 @@ async fn test_handle_append_result_stale_term_ignored() { }; let result = context .state - .handle_append_result(2, Ok(stale_response), &context.raft_context, &role_tx) + .handle_append_result( + 2, + Ok(stale_response), + &context.raft_context, + &internal_event_tx, + ) .await; assert!(result.is_ok(), "stale term response returns Ok (ignored)"); @@ -2076,7 +2128,7 @@ async fn test_handle_append_result_stale_term_ignored() { "commit unchanged — stale term response ignored" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no event emitted for stale response" ); } @@ -2104,7 +2156,7 @@ async fn test_handle_append_result_rpc_failure_ignored() { // No mock expectations — Err triggers early return before any handler call - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); let result = context .state @@ -2112,7 +2164,7 @@ async fn test_handle_append_result_rpc_failure_ignored() { 2, Err(crate::Error::Fatal("connection refused".to_string())), &context.raft_context, - &role_tx, + &internal_event_tx, ) .await; @@ -2126,7 +2178,7 @@ async fn test_handle_append_result_rpc_failure_ignored() { "commit unchanged — RPC failure ignored" ); assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "no event emitted for RPC failure" ); } @@ -2224,7 +2276,7 @@ async fn test_stale_conflict_ack_does_not_regress_next_index() { state.match_index.insert(peer_id, 100); state.next_index.insert(peer_id, 101); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Stale CONFLICT from Batch B: follower's log was empty when it arrived. let stale_conflict = AppendEntriesResponse { @@ -2237,7 +2289,12 @@ async fn test_stale_conflict_ack_does_not_regress_next_index() { }; let result = state - .handle_append_result(peer_id, Ok(stale_conflict), &context_inner, &role_tx) + .handle_append_result( + peer_id, + Ok(stale_conflict), + &context_inner, + &internal_event_tx, + ) .await; assert!(result.is_ok(), "stale conflict must not return an error"); @@ -2368,9 +2425,14 @@ async fn test_stale_conflict_floor_guard_prevents_regression() { ) .await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let result = state - .handle_append_result(follower2_id, Ok(stale_conflict), &context_inner, &role_tx) + .handle_append_result( + follower2_id, + Ok(stale_conflict), + &context_inner, + &internal_event_tx, + ) .await; assert!(result.is_ok(), "stale conflict must not return an error"); @@ -2399,10 +2461,15 @@ async fn test_conflict_response_resets_speculative_next_index() { ) .await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let result = state - .handle_append_result(follower2_id, Ok(stale_conflict), &context_inner, &role_tx) + .handle_append_result( + follower2_id, + Ok(stale_conflict), + &context_inner, + &internal_event_tx, + ) .await; assert!(result.is_ok(), "stale conflict must not return an error"); diff --git a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs index 511fbaf3..3b9b8d07 100644 --- a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs @@ -73,10 +73,10 @@ async fn setup_single_voter( async fn test_single_voter_commit_uses_last_entry_id_not_durable() { // last_entry_id=5: entries 4-5 arrived in memory during the IO flush of 1-3 let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // IO flushed only entries 1-3 (durable=3), but log has entries 1-5 in memory - state.handle_log_flushed(3, &ctx, &role_tx).await; + state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), @@ -90,9 +90,9 @@ async fn test_single_voter_commit_uses_last_entry_id_not_durable() { #[tokio::test] async fn test_single_voter_commit_when_durable_equals_last_entry_id() { let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - state.handle_log_flushed(5, &ctx, &role_tx).await; + state.handle_log_flushed(5, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), @@ -109,10 +109,10 @@ async fn test_single_voter_commit_when_durable_equals_last_entry_id() { #[tokio::test] async fn test_single_voter_pipelining_across_io_batches() { let (mut state, ctx, last_entry_id) = setup_single_voter(7).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // IO batch 1: flushed 1-3, memory has 1-7 - state.handle_log_flushed(3, &ctx, &role_tx).await; + state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), 7, @@ -121,7 +121,7 @@ async fn test_single_voter_pipelining_across_io_batches() { // IO batch 2: flushed 4-7, memory now has 1-10 last_entry_id.store(10, Ordering::Relaxed); - state.handle_log_flushed(7, &ctx, &role_tx).await; + state.handle_log_flushed(7, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), 10, @@ -133,14 +133,14 @@ async fn test_single_voter_pipelining_across_io_batches() { #[tokio::test] async fn test_single_voter_no_commit_when_nothing_new() { let (mut state, ctx, _last_entry_id) = setup_single_voter(3).await; - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First flush advances commit to 3 - state.handle_log_flushed(3, &ctx, &role_tx).await; + state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!(state.commit_index(), 3); // Second flush with same last_entry_id=3: no new entries → no commit advance - state.handle_log_flushed(3, &ctx, &role_tx).await; + state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), 3, diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs index 2c993ddf..25a9fe1f 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_test.rs @@ -9,7 +9,7 @@ use std::sync::atomic::Ordering; use crate::SnapshotError; use crate::config::RaftNodeConfig as CoreRaftNodeConfig; -use crate::event::RaftEvent; +use crate::event::InboundEvent; use crate::raft_role::leader_state::LeaderState; use crate::role_state::RaftRoleState; use crate::test_utils::mock::{MockBuilder, MockTypeConfig}; @@ -48,18 +48,18 @@ async fn test_create_snapshot_event_starts_and_ignores_duplicates() { // Initially, no snapshot in progress assert!(!state.snapshot_in_progress.load(Ordering::SeqCst)); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First event should set the flag and spawn the task state - .handle_create_snapshot(&context, &role_tx) + .handle_create_snapshot(&context, &internal_event_tx) .await .expect("Should start snapshot creation"); assert!(state.snapshot_in_progress.load(Ordering::SeqCst)); // Second event should be ignored (flag still set) state - .handle_create_snapshot(&context, &role_tx) + .handle_create_snapshot(&context, &internal_event_tx) .await .expect("Should ignore duplicate snapshot creation"); // Still true, no panic, no duplicate task @@ -90,7 +90,7 @@ async fn test_snapshot_in_progress_flag_reset_on_created_event() { let mut state = LeaderState::::new(1, context.node_config()); state.snapshot_in_progress.store(true, Ordering::SeqCst); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // Success case let _ = state @@ -103,7 +103,7 @@ async fn test_snapshot_in_progress_flag_reset_on_created_event() { PathBuf::from("/tmp/fake"), )), &context, - &role_tx, + &internal_event_tx, ) .await; assert!(!state.snapshot_in_progress.load(Ordering::SeqCst)); @@ -114,7 +114,7 @@ async fn test_snapshot_in_progress_flag_reset_on_created_event() { .handle_snapshot_created( Err(SnapshotError::OperationFailed("fail".into()).into()), &context, - &role_tx, + &internal_event_tx, ) .await; assert!(!state.snapshot_in_progress.load(Ordering::SeqCst)); @@ -140,10 +140,10 @@ async fn test_create_snapshot_event_is_non_blocking() { let context = MockBuilder::new(graceful_rx).build_context(); let mut state = LeaderState::::new(1, context.node_config()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let start = std::time::Instant::now(); - let _ = state.handle_create_snapshot(&context, &role_tx).await; + let _ = state.handle_create_snapshot(&context, &internal_event_tx).await; let elapsed = start.elapsed(); // Should return quickly (not wait for snapshot) @@ -242,12 +242,12 @@ async fn test_stream_snapshot_rejects_when_no_metadata() { mpsc::channel::>(4); let (startup_tx, startup_rx) = tokio::sync::oneshot::channel::>(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state - .handle_raft_event( - RaftEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), + .handle_inbound_event( + InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), &context, - role_tx, + internal_event_tx, ) .await .expect("handler must not return Err"); @@ -338,12 +338,12 @@ async fn test_stream_snapshot_confirms_startup_when_metadata_exists() { mpsc::channel::>(4); let (startup_tx, startup_rx) = tokio::sync::oneshot::channel::>(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state - .handle_raft_event( - RaftEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), + .handle_inbound_event( + InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), &context, - role_tx, + internal_event_tx, ) .await .expect("handler must not return Err"); diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index 50344adb..f098d803 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -7,7 +7,7 @@ //! 3. Skip subsequent `Snapshot` tasks that arrive while in progress. //! 4. Skip `Append` tasks that arrive while snapshot is in progress //! (they are meaningless until the peer has the snapshot base). -//! 5. After transfer completes, emit `RoleEvent::SnapshotPushCompleted`. +//! 5. After transfer completes, emit `InternalEvent::SnapshotPushCompleted`. use std::sync::Arc; use std::time::Duration; @@ -25,7 +25,7 @@ use crate::MockRaftLog; use crate::MockStateMachine; use crate::MockTransport; use crate::RaftRequestWithSignal; -use crate::event::RoleEvent; +use crate::event::InternalEvent; use crate::maybe_clone_oneshot::{MaybeCloneOneshot, RaftOneshot}; use crate::raft_role::leader_state::LeaderState; use crate::raft_role::role_state::RaftRoleState; @@ -109,11 +109,11 @@ fn one_entry_batch() -> std::collections::VecDeque { /// Worker routes snapshot task: when `prepare_batch_requests` returns a /// snapshot target, the worker receives `ReplicationTask::Snapshot` and -/// emits `RoleEvent::SnapshotPushCompleted` once the transfer finishes. +/// emits `InternalEvent::SnapshotPushCompleted` once the transfer finishes. /// /// # Scenario /// - `prepare_batch_requests` returns snapshot_targets = [2], append_requests = [] -/// - Expected: `RoleEvent::SnapshotPushCompleted { peer_id: 2, success: true }` arrives +/// - Expected: `InternalEvent::SnapshotPushCompleted { peer_id: 2, success: true }` arrives /// /// # Before fix (will FAIL) /// `ReplicationTask` doesn't exist; worker only handles AppendEntries. @@ -170,19 +170,19 @@ async fn test_worker_handles_snapshot_task_and_emits_completed_event() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.init_cluster_metadata(&ctx.membership).await.unwrap(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // Allow worker task to run and complete the (mock) snapshot transfer. tokio::time::sleep(Duration::from_millis(100)).await; - let event = role_rx + let event = internal_event_rx .try_recv() .expect("SnapshotPushCompleted should arrive after snapshot transfer"); assert!( matches!( event, - RoleEvent::SnapshotPushCompleted { + InternalEvent::SnapshotPushCompleted { peer_id: 2, success: true } @@ -256,12 +256,12 @@ async fn test_worker_skips_duplicate_snapshot_while_in_progress() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.init_cluster_metadata(&ctx.membership).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First batch: snapshot task sent to worker (transfer begins). - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // Second batch arrives before first snapshot completes. - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // Allow worker time to finish processing. tokio::time::sleep(Duration::from_millis(200)).await; @@ -346,12 +346,12 @@ async fn test_worker_skips_append_while_snapshot_in_progress() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.init_cluster_metadata(&ctx.membership).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First batch: snapshot task. - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // Second batch: append task arrives while snapshot is still running. - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // Wait for snapshot to finish. tokio::time::sleep(Duration::from_millis(200)).await; diff --git a/d-engine-core/src/raft_role/leader_state_test/stale_learner_deadline_test.rs b/d-engine-core/src/raft_role/leader_state_test/stale_learner_deadline_test.rs index 1567445e..f917721e 100644 --- a/d-engine-core/src/raft_role/leader_state_test/stale_learner_deadline_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/stale_learner_deadline_test.rs @@ -38,17 +38,17 @@ async fn setup_with_threshold( ) -> ( LeaderState, crate::RaftContext, - mpsc::UnboundedSender, - mpsc::Sender, + mpsc::UnboundedSender, + mpsc::Sender, ) { let (_shutdown_tx, shutdown_rx) = watch::channel(()); let mut cfg = node_config("/tmp/stale_learner_deadline_test"); cfg.raft.membership.promotion.stale_learner_threshold = threshold; let ctx = MockBuilder::new(shutdown_rx).with_node_config(cfg).build_context(); let state = LeaderState::::new(1, ctx.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (raft_tx, _raft_rx) = mpsc::channel(1); - (state, ctx, role_tx, raft_tx) + (state, ctx, internal_event_tx, raft_tx) } // ============================================================================ @@ -57,7 +57,8 @@ async fn setup_with_threshold( #[tokio::test] async fn test_stale_check_deadline_none_when_queue_empty() { - let (leader, _ctx, _role_tx, _raft_tx) = setup_with_threshold(Duration::from_secs(30)).await; + let (leader, _ctx, _internal_event_tx, _raft_tx) = + setup_with_threshold(Duration::from_secs(30)).await; assert!( leader.stale_check_deadline.is_none(), @@ -72,7 +73,7 @@ async fn test_stale_check_deadline_none_when_queue_empty() { #[tokio::test] async fn test_stale_check_deadline_set_on_first_enqueue() { let threshold = Duration::from_secs(30); - let (mut leader, _ctx, _role_tx, _raft_tx) = setup_with_threshold(threshold).await; + let (mut leader, _ctx, _internal_event_tx, _raft_tx) = setup_with_threshold(threshold).await; let before = Instant::now(); leader.pending_promotions.push_back(PendingPromotion::new(10, Instant::now())); @@ -94,7 +95,7 @@ async fn test_stale_check_deadline_set_on_first_enqueue() { #[tokio::test] async fn test_stale_check_deadline_unchanged_on_second_enqueue() { let threshold = Duration::from_secs(30); - let (mut leader, _ctx, _role_tx, _raft_tx) = setup_with_threshold(threshold).await; + let (mut leader, _ctx, _internal_event_tx, _raft_tx) = setup_with_threshold(threshold).await; // Enqueue first learner let t0 = Instant::now(); @@ -121,7 +122,7 @@ async fn test_stale_check_deadline_unchanged_on_second_enqueue() { #[tokio::test] async fn test_tick_skips_stale_check_before_deadline() { let threshold = Duration::from_secs(3600); // Far future - let (mut leader, mut ctx, role_tx, raft_tx) = setup_with_threshold(threshold).await; + let (mut leader, mut ctx, internal_event_tx, raft_tx) = setup_with_threshold(threshold).await; let mut membership = MockMembership::new(); // get_node_status must NOT be called — stale check should be skipped @@ -133,7 +134,7 @@ async fn test_tick_skips_stale_check_before_deadline() { leader.refresh_stale_deadline(threshold); // Call tick() — deadline is far in the future, stale check must be skipped - let result = leader.tick(&role_tx, &raft_tx, &ctx).await; + let result = leader.tick(&internal_event_tx, &raft_tx, &ctx).await; assert!(result.is_ok()); // If MockMembership::get_node_status were called, mockall would panic — test passes by not panicking } @@ -187,7 +188,7 @@ async fn test_tick_removes_stale_learner_when_deadline_fires() { let node_config = ctx.node_config(); let mut leader = LeaderState::::new(1, node_config); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (raft_tx, _raft_rx) = mpsc::channel(1); // Enqueue learner with a ready_since in the past so it's already stale @@ -196,7 +197,7 @@ async fn test_tick_removes_stale_learner_when_deadline_fires() { // Deadline = past + 1ms — already fired leader.stale_check_deadline = Some(past + threshold); - let result = leader.tick(&role_tx, &raft_tx, &ctx).await; + let result = leader.tick(&internal_event_tx, &raft_tx, &ctx).await; assert!(result.is_ok()); // Verify a config change (BatchRemove or RemoveNode) was proposed for node 10 @@ -214,7 +215,7 @@ async fn test_tick_removes_stale_learner_when_deadline_fires() { #[tokio::test] async fn test_drain_batch_clears_deadline_when_queue_empty() { let threshold = Duration::from_secs(30); - let (mut leader, _ctx, _role_tx, _raft_tx) = setup_with_threshold(threshold).await; + let (mut leader, _ctx, _internal_event_tx, _raft_tx) = setup_with_threshold(threshold).await; leader.pending_promotions.push_back(PendingPromotion::new(10, Instant::now())); leader.refresh_stale_deadline(threshold); @@ -237,7 +238,7 @@ async fn test_drain_batch_clears_deadline_when_queue_empty() { #[tokio::test] async fn test_drain_batch_refreshes_deadline_to_new_front() { let threshold = Duration::from_secs(30); - let (mut leader, _ctx, _role_tx, _raft_tx) = setup_with_threshold(threshold).await; + let (mut leader, _ctx, _internal_event_tx, _raft_tx) = setup_with_threshold(threshold).await; let t0 = Instant::now() - Duration::from_secs(5); // older let t1 = Instant::now() - Duration::from_secs(2); // newer (becomes new front after drain) diff --git a/d-engine-core/src/raft_role/leader_state_test/state_management_test.rs b/d-engine-core/src/raft_role/leader_state_test/state_management_test.rs index 351a6fa1..1f9780ba 100644 --- a/d-engine-core/src/raft_role/leader_state_test/state_management_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/state_management_test.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use tokio::sync::{mpsc, watch}; use tracing_test::traced_test; -use crate::event::RaftEvent; +use crate::event::InboundEvent; use crate::maybe_clone_oneshot::RaftOneshot; use crate::raft_role::leader_state::LeaderState; use crate::role_state::RaftRoleState; @@ -302,12 +302,12 @@ async fn test_handle_discover_leader_success() { node_id: 100, requester_address: "127.0.0.1:8080".to_string(), }; - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state - .handle_raft_event(raft_event, &context, role_tx) + .handle_inbound_event(inbound_event, &context, internal_event_tx) .await .expect("Should handle successfully"); @@ -355,12 +355,12 @@ async fn test_handle_discover_leader_metadata_not_found() { node_id: 100, requester_address: "127.0.0.1:8080".to_string(), }; - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state - .handle_raft_event(raft_event, &context, role_tx) + .handle_inbound_event(inbound_event, &context, internal_event_tx) .await .expect("Should panic during handling"); } @@ -412,12 +412,12 @@ async fn test_handle_discover_leader_different_terms() { node_id: 100, requester_address: "127.0.0.1:8080".to_string(), }; - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state - .handle_raft_event(raft_event, &context, role_tx) + .handle_inbound_event(inbound_event, &context, internal_event_tx) .await .expect("Should handle successfully"); @@ -472,12 +472,12 @@ async fn test_handle_discover_leader_invalid_node_id() { node_id: 0, requester_address: "127.0.0.1:8080".to_string(), }; - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); state - .handle_raft_event(raft_event, &context, role_tx) + .handle_inbound_event(inbound_event, &context, internal_event_tx) .await .expect("Should handle successfully"); diff --git a/d-engine-core/src/raft_role/leader_state_test/worker_lifecycle_test.rs b/d-engine-core/src/raft_role/leader_state_test/worker_lifecycle_test.rs index 1bf661c4..8b3477fe 100644 --- a/d-engine-core/src/raft_role/leader_state_test/worker_lifecycle_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/worker_lifecycle_test.rs @@ -24,7 +24,7 @@ use crate::MockMembership; use crate::MockRaftLog; use crate::MockTransport; use crate::RaftRequestWithSignal; -use crate::event::RoleEvent; +use crate::event::InternalEvent; use crate::maybe_clone_oneshot::{MaybeCloneOneshot, RaftOneshot}; use crate::raft_role::leader_state::LeaderState; use crate::raft_role::role_state::RaftRoleState; @@ -158,8 +158,8 @@ async fn test_worker_spawned_on_first_request() { "precondition: no workers before first batch" ); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // A worker must have been inserted for peer 2. assert_eq!( @@ -238,10 +238,10 @@ async fn test_worker_reused_on_subsequent_requests() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.init_cluster_metadata(&ctx.membership).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First batch — spawns the worker. - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); assert_eq!( state.worker_count_for_test(), 1, @@ -249,7 +249,7 @@ async fn test_worker_reused_on_subsequent_requests() { ); // Second batch — must reuse the same worker, not spawn a second one. - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); assert_eq!( state.worker_count_for_test(), 1, @@ -263,11 +263,11 @@ async fn test_worker_reused_on_subsequent_requests() { /// - A dead worker handle is injected for peer 2 via `inject_dead_worker_for_test`. /// - `process_batch` returns one request for peer 2. /// - `send_to_worker_or_spawn` detects the dead channel and rebuilds the worker. -/// - The new worker calls `send_append_request` and emits `RoleEvent::AppendResult`. +/// - The new worker calls `send_append_request` and emits `InternalEvent::AppendResult`. /// /// # Guarantees checked /// - `worker_count_for_test() == 1` after the rebuild (old dead handle replaced). -/// - `RoleEvent::AppendResult` arrives on `role_rx`, confirming the request was delivered. +/// - `InternalEvent::AppendResult` arrives on `internal_event_rx`, confirming the request was delivered. #[tokio::test] #[traced_test] async fn test_worker_rebuilt_after_death() { @@ -288,7 +288,7 @@ async fn test_worker_rebuilt_after_death() { }); // Transport: allow async calls from the rebuilt worker. We verify delivery - // indirectly via AppendResult on role_rx rather than mock call count. + // indirectly via AppendResult on internal_event_rx rather than mock call count. let mut transport = MockTransport::::new(); transport .expect_send_append_request() @@ -330,8 +330,8 @@ async fn test_worker_rebuilt_after_death() { "precondition: dead handle present" ); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // The dead handle must be replaced by a live one (still 1, not 0 or 2). assert_eq!( @@ -344,9 +344,11 @@ async fn test_worker_rebuilt_after_death() { tokio::time::sleep(std::time::Duration::from_millis(50)).await; // The rebuilt worker must relay AppendResult back to the Raft loop. - let event = role_rx.try_recv().expect("AppendResult should arrive from rebuilt worker"); + let event = internal_event_rx + .try_recv() + .expect("AppendResult should arrive from rebuilt worker"); assert!( - matches!(event, RoleEvent::AppendResult { follower_id: 2, .. }), + matches!(event, InternalEvent::AppendResult { follower_id: 2, .. }), "expected AppendResult for peer 2, got {event:?}" ); } @@ -363,7 +365,7 @@ async fn test_worker_rebuilt_after_death() { /// # Guarantees checked /// - After the handle is dropped, `open_replication_stream` call count stops increasing. /// This proves the `if task_rx.is_closed() { return; }` guard fires correctly, -/// preventing an infinite reconnect loop that would flood the Raft event loop with +/// preventing an infinite reconnect loop that would flood the inbound event loop with /// zombie signals and block heartbeats (regression: zombie detection + dead peer). #[tokio::test] #[traced_test] @@ -418,8 +420,8 @@ async fn test_replication_worker_exits_when_handle_dropped() { let mut state = LeaderState::::new(1, ctx.node_config.clone()); state.init_cluster_metadata(&ctx.membership).await.unwrap(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - state.process_batch(one_entry_batch(), &role_tx, &ctx).await.unwrap(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); // Wait until the worker has attempted at least one connection (entered inner loop). tokio::time::timeout(std::time::Duration::from_secs(5), first_attempt_rx.recv()) diff --git a/d-engine-core/src/raft_role/learner_state.rs b/d-engine-core/src/raft_role/learner_state.rs index 04d7f07c..0e580aed 100644 --- a/d-engine-core/src/raft_role/learner_state.rs +++ b/d-engine-core/src/raft_role/learner_state.rs @@ -6,16 +6,16 @@ use super::follower_state::FollowerState; use super::role_state::RaftRoleState; use super::role_state::check_and_trigger_snapshot; use crate::ConsensusError; +use crate::InboundEvent; +use crate::InternalEvent; use crate::Membership; use crate::MembershipError; use crate::NetworkError; use crate::PurgeExecutor; use crate::RaftContext; -use crate::RaftEvent; use crate::RaftLog; use crate::RaftNodeConfig; use crate::Result; -use crate::RoleEvent; use crate::StateMachineHandler; use crate::StateTransitionError; use crate::Transport; @@ -172,25 +172,25 @@ impl RaftRoleState for LearnerState { async fn tick( &mut self, - _role_event_tx: &mpsc::UnboundedSender, - _raft_tx: &mpsc::Sender, + _internal_event_tx: &mpsc::UnboundedSender, + _raft_tx: &mpsc::Sender, _ctx: &RaftContext, ) -> Result<()> { Ok(()) } - async fn handle_raft_event( + async fn handle_inbound_event( &mut self, - raft_event: RaftEvent, + inbound_event: InboundEvent, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, ) -> Result<()> { let state_snapshot = self.state_snapshot(); let my_term = self.current_term(); - match raft_event { - RaftEvent::ReceiveVoteRequest(vote_request, sender) => { - info!("handle_raft_event::ReceiveVoteRequest. Learner cannot vote."); + match inbound_event { + InboundEvent::ReceiveVoteRequest(vote_request, sender) => { + info!("handle_inbound_event::ReceiveVoteRequest. Learner cannot vote."); // 1. Update term FIRST if needed if vote_request.term > my_term { self.update_current_term(vote_request.term); @@ -216,7 +216,7 @@ impl RaftRoleState for LearnerState { })?; } - RaftEvent::ClusterConf(_metadata_request, sender) => { + InboundEvent::ClusterConf(_metadata_request, sender) => { debug!("Learner receive ClusterConf request..."); sender .send(Err(Status::permission_denied( @@ -228,7 +228,7 @@ impl RaftRoleState for LearnerState { })?; } - RaftEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { + InboundEvent::ClusterConfUpdate(cluste_conf_change_request, sender) => { let current_conf_version = ctx.membership().get_cluster_conf_version().await; let current_leader_id = self.shared_state().current_leader(); @@ -270,18 +270,18 @@ impl RaftRoleState for LearnerState { })?; } - RaftEvent::AppendEntries(append_entries_request, sender) => { + InboundEvent::AppendEntries(append_entries_request, sender) => { self.handle_append_entries_request_workflow( append_entries_request, sender, ctx, - role_tx, + internal_event_tx, &state_snapshot, ) .await?; } - RaftEvent::InstallSnapshotChunk(stream, sender) => { + InboundEvent::InstallSnapshotChunk(stream, sender) => { // ack_tx is passed to process_snapshot_stream for per-chunk validation. // In push mode the receiver is never read, so we drain it in a background // task to prevent backpressure: process_snapshot_stream awaits each send, @@ -312,7 +312,7 @@ impl RaftRoleState for LearnerState { match snap_result { Err(e) => { - error!(?e, "Learner handle RaftEvent::InstallSnapshotChunk"); + error!(?e, "Learner handle InboundEvent::InstallSnapshotChunk"); return Err(e); } Ok(()) => { @@ -338,7 +338,7 @@ impl RaftRoleState for LearnerState { } } - RaftEvent::JoinCluster(_join_request, sender) => { + InboundEvent::JoinCluster(_join_request, sender) => { sender .send(Err(Status::permission_denied( "Learner should not receive JoinCluster event.", @@ -352,15 +352,15 @@ impl RaftRoleState for LearnerState { current_role: "Learner", required_role: "Leader", context: format!( - "Learner node {} receives RaftEvent::JoinCluster", + "Learner node {} receives InboundEvent::JoinCluster", ctx.node_id ), } .into()); } - RaftEvent::DiscoverLeader(request, sender) => { - debug!(?request, "Learner::RaftEvent::DiscoverLeader"); + InboundEvent::DiscoverLeader(request, sender) => { + debug!(?request, "Learner::InboundEvent::DiscoverLeader"); sender .send(Err(Status::permission_denied( "Learner should not response DiscoverLeader event.", @@ -373,8 +373,8 @@ impl RaftRoleState for LearnerState { return Ok(()); } - RaftEvent::StreamSnapshot(_ack_rx, _chunk_tx, startup_tx) => { - debug!("Learner::RaftEvent::StreamSnapshot"); + InboundEvent::StreamSnapshot(_ack_rx, _chunk_tx, startup_tx) => { + debug!("Learner::InboundEvent::StreamSnapshot"); warn!("Learner should not receive StreamSnapshot event."); if let Err(e) = startup_tx.send(Err(Status::failed_precondition("Not the leader"))) { @@ -386,7 +386,7 @@ impl RaftRoleState for LearnerState { return Ok(()); } - RaftEvent::FatalError { source, error } => { + InboundEvent::FatalError { source, error } => { error!("[Learner] Fatal error from {}: {}", source, error); return Err(crate::Error::Fatal(format!( "Fatal error from {source}: {error}" @@ -513,7 +513,7 @@ impl RaftRoleState for LearnerState { last_index: u64, _results: Vec, ctx: &crate::RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> crate::Result<()> { // Per Raft §7: each server takes snapshots independently. check_and_trigger_snapshot( @@ -521,7 +521,7 @@ impl RaftRoleState for LearnerState { Learner as i32, self.current_term(), ctx, - role_tx, + internal_event_tx, ) } @@ -531,7 +531,7 @@ impl RaftRoleState for LearnerState { async fn handle_create_snapshot( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Prevent duplicate snapshot creation if self.snapshot_in_progress.load(Ordering::Acquire) { @@ -543,11 +543,11 @@ impl RaftRoleState for LearnerState { let state_machine_handler = ctx.state_machine_handler().clone(); // Use spawn to perform snapshot creation in the background - let role_tx = role_tx.clone(); + let internal_event_tx = internal_event_tx.clone(); tokio::spawn(async move { let result = state_machine_handler.create_snapshot().await; info!("SnapshotCreated event will be processed in another event thread"); - if let Err(e) = role_tx.send(RoleEvent::SnapshotCreated(result)) { + if let Err(e) = internal_event_tx.send(InternalEvent::SnapshotCreated(result)) { error!("Follower failed to send snapshot creation result: {}", e); } }); @@ -563,7 +563,7 @@ impl RaftRoleState for LearnerState { &mut self, result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Reset snapshot_in_progress flag self.snapshot_in_progress.store(false, Ordering::SeqCst); @@ -602,7 +602,7 @@ impl RaftRoleState for LearnerState { async fn handle_membership_applied( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Check if this learner has been promoted to Voter let my_id = self.node_id(); @@ -620,7 +620,7 @@ impl RaftRoleState for LearnerState { ); // Transition to Follower role - role_tx.send(RoleEvent::BecomeFollower(None)).map_err(|e| { + internal_event_tx.send(InternalEvent::BecomeFollower(None)).map_err(|e| { error!("Failed to send BecomeFollower event: {e:?}"); NetworkError::SingalSendFailed(format!("{:?}", e)) })?; @@ -640,7 +640,7 @@ impl RaftRoleState for LearnerState { } // Stale in-flight events — these are Leader-only operations that may arrive - // after a step-down due to role_tx (unbounded, P2) race with BecomeFollower. + // after a step-down due to internal_event_tx (unbounded, P2) race with BecomeFollower. // Both orderings are valid; Learner silently ignores them rather than erroring. fn handle_log_purge_completed( @@ -653,7 +653,7 @@ impl RaftRoleState for LearnerState { async fn handle_promote_ready_learners( &mut self, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Ok(()) } diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index fde74f08..17c20fd4 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -1,13 +1,13 @@ use crate::ClientCmd; use crate::Error; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MaybeCloneOneshot; use crate::MockBuilder; use crate::MockMembership; use crate::MockStateMachineHandler; use crate::NewCommitData; -use crate::RaftEvent; use crate::RaftOneshot; -use crate::RoleEvent; use crate::client::ClientReadRequest; use crate::client::ClientResponsePayload; use crate::client::ClientWriteRequest; @@ -88,12 +88,12 @@ async fn test_learner_tick_succeeds() { let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (event_tx, _event_rx) = mpsc::channel(1); // Action: Tick assert!( - state.tick(&role_tx, &event_tx, &context).await.is_ok(), + state.tick(&internal_event_tx, &event_tx, &context).await.is_ok(), "Learner tick should succeed" ); } @@ -108,12 +108,12 @@ async fn test_learner_tick_succeeds() { /// Expected: /// - Returns response with vote_granted=false /// - Updates current_term to request term (11) -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// /// This validates Raft rule: all nodes update term when seeing higher term, /// but learners never grant votes. /// -/// Original: test_handle_raft_event_case1 +/// Original: test_handle_inbound_event_case1 #[tokio::test] async fn test_learner_rejects_vote_request_updates_term() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -124,8 +124,8 @@ async fn test_learner_rejects_vote_request_updates_term() { let request_term = term_before + 10; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ReceiveVoteRequest( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ReceiveVoteRequest( VoteRequest { term: request_term, candidate_id: 2, @@ -137,8 +137,11 @@ async fn test_learner_rejects_vote_request_updates_term() { // Action: Handle VoteRequest assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); // Verify: Term updated @@ -161,11 +164,11 @@ async fn test_learner_rejects_vote_request_updates_term() { /// /// Expected: /// - Returns Status error with Code::PermissionDenied -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// /// This validates that learners redirect metadata requests to leader. /// -/// Original: test_handle_raft_event_case2 +/// Original: test_handle_inbound_event_case2 #[tokio::test] async fn test_learner_rejects_cluster_conf_request() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -174,13 +177,16 @@ async fn test_learner_rejects_cluster_conf_request() { let mut state = LearnerState::::new(1, context.node_config.clone()); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConf(MetadataRequest {}, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConf(MetadataRequest {}, resp_tx); // Action: Handle ClusterConf assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); // Verify: PermissionDenied @@ -201,12 +207,12 @@ async fn test_learner_rejects_cluster_conf_request() { /// Expected: /// - Returns response with success=true /// - error_code = Unspecified -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// /// This validates that learners can receive and apply cluster /// configuration updates from leader. /// -/// Original: test_handle_raft_event_case3 +/// Original: test_handle_inbound_event_case3 #[tokio::test] async fn test_learner_handles_cluster_conf_update_success() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -232,8 +238,8 @@ async fn test_learner_handles_cluster_conf_update_success() { let mut state = LearnerState::::new(1, context.node_config.clone()); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let raft_event = RaftEvent::ClusterConfUpdate( + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::ClusterConfUpdate( ClusterConfChangeRequest { id: 2, // Leader ID term: 1, @@ -245,8 +251,11 @@ async fn test_learner_handles_cluster_conf_update_success() { // Action: Handle ClusterConfUpdate assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); // Verify: Success response @@ -275,7 +284,7 @@ async fn test_learner_handles_cluster_conf_update_success() { /// - Updates commit_index /// - Returns AppendEntriesResponse with success=true /// -/// Original: test_handle_raft_event_case4_1 +/// Original: test_handle_inbound_event_case4_1 #[tokio::test] async fn test_learner_handles_append_entries_success() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -317,24 +326,27 @@ async fn test_learner_handles_append_entries_success() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); // Verify: LeaderDiscovered event assert!(matches!( - role_rx.try_recv().unwrap(), - crate::RoleEvent::LeaderDiscovered(5, _) + internal_event_rx.try_recv().unwrap(), + crate::InternalEvent::LeaderDiscovered(5, _) )); // Verify: NotifyNewCommitIndex event assert!(matches!( - role_rx.try_recv().unwrap(), - crate::RoleEvent::NotifyNewCommitIndex(_) + internal_event_rx.try_recv().unwrap(), + crate::InternalEvent::NotifyNewCommitIndex(_) )); assert_eq!(state.current_term(), leader_term); @@ -354,7 +366,7 @@ async fn test_learner_handles_append_entries_success() { /// - Term unchanged /// - Returns AppendEntriesResponse with is_higher_term=true /// -/// Original: test_handle_raft_event_case4_2 +/// Original: test_handle_inbound_event_case4_2 #[tokio::test] async fn test_learner_rejects_append_entries_stale_term() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -378,15 +390,21 @@ async fn test_learner_rejects_append_entries_stale_term() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); - assert!(role_rx.try_recv().is_err(), "No events should be sent"); + assert!( + internal_event_rx.try_recv().is_err(), + "No events should be sent" + ); assert_eq!(state.current_term(), learner_term); let response = resp_rx.recv().await.unwrap().unwrap(); @@ -403,9 +421,9 @@ async fn test_learner_rejects_append_entries_stale_term() { /// - Sends LeaderDiscovered event (leader is valid) /// - Updates term /// - Returns AppendEntriesResponse with success=false -/// - handle_raft_event returns Err() +/// - handle_inbound_event returns Err() /// -/// Original: test_handle_raft_event_case4_3 +/// Original: test_handle_inbound_event_case4_3 #[tokio::test] async fn test_learner_handles_append_entries_handler_error() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -435,19 +453,22 @@ async fn test_learner_handles_append_entries_handler_error() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_err(), - "handle_raft_event should return error" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_err(), + "handle_inbound_event should return error" ); assert!(matches!( - role_rx.try_recv().unwrap(), - crate::RoleEvent::LeaderDiscovered(5, _) + internal_event_rx.try_recv().unwrap(), + crate::InternalEvent::LeaderDiscovered(5, _) )); - assert!(role_rx.try_recv().is_err()); + assert!(internal_event_rx.try_recv().is_err()); assert_eq!(state.current_term(), leader_term); @@ -468,7 +489,7 @@ async fn test_learner_handles_append_entries_handler_error() { /// Expected: /// - Returns response with error_code = NOT_LEADER /// -/// Original: test_handle_raft_event_case5 +/// Original: test_handle_inbound_event_case5 #[tokio::test] async fn test_learner_rejects_client_write_request() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -506,7 +527,7 @@ async fn test_learner_rejects_client_write_request() { /// Expected: /// - Returns response with error_code = NOT_LEADER /// -/// Original: test_handle_raft_event_case6 +/// Original: test_handle_inbound_event_case6 #[tokio::test] async fn test_learner_rejects_client_read_request() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -533,7 +554,7 @@ async fn test_learner_rejects_client_read_request() { assert!(err.message().contains("Not leader")); } /// -/// Original: test_handle_raft_event_case10 +/// Original: test_handle_inbound_event_case10 #[tokio::test] async fn test_learner_rejects_join_cluster() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -548,12 +569,15 @@ async fn test_learner_rejects_join_cluster() { address: "127.0.0.1:9090".to_string(), }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::JoinCluster(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::JoinCluster(request, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_err(), - "handle_raft_event should return error" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_err(), + "handle_inbound_event should return error" ); let response = resp_rx.recv().await.expect("should receive response"); @@ -570,9 +594,9 @@ async fn test_learner_rejects_join_cluster() { /// /// Expected: /// - Returns Status error with Code::PermissionDenied -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// -/// Original: test_handle_raft_event_case11 +/// Original: test_handle_inbound_event_case11 #[tokio::test] async fn test_learner_rejects_leader_discovery() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -585,12 +609,15 @@ async fn test_learner_rejects_leader_discovery() { requester_address: "127.0.0.1:9090".to_string(), }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::DiscoverLeader(request, resp_tx); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); assert!( - state.handle_raft_event(raft_event, &context, role_tx).await.is_ok(), - "handle_raft_event should succeed" + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" ); let response = resp_rx.recv().await.expect("should receive response"); @@ -1041,7 +1068,7 @@ async fn test_join_cluster_succeeds_large_cluster() { /// /// Expected: /// - Sends BecomeFollower event -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// /// This validates learner promotion mechanism. /// @@ -1069,18 +1096,21 @@ async fn test_learner_promotion_on_membership_applied() { context.membership = Arc::new(mock_membership); let mut state = LearnerState::::new(3, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = state.handle_membership_applied(&context, &role_tx).await; + let result = state.handle_membership_applied(&context, &internal_event_tx).await; assert!(result.is_ok(), "MembershipApplied should succeed"); - let role_event = tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()) - .await - .expect("Should receive event within timeout") - .expect("Channel should not be closed"); + let internal_event = tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await + .expect("Should receive event within timeout") + .expect("Channel should not be closed"); - match role_event { - crate::RoleEvent::BecomeFollower(leader_id) => { + match internal_event { + crate::InternalEvent::BecomeFollower(leader_id) => { assert_eq!(leader_id, None, "Should not specify leader on promotion"); } other => panic!("Expected BecomeFollower event, got: {other:?}"), @@ -1095,7 +1125,7 @@ async fn test_learner_promotion_on_membership_applied() { /// /// Expected: /// - No role transition event -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// /// Original: test_learner_stays_learner_on_membership_applied #[tokio::test] @@ -1121,13 +1151,16 @@ async fn test_learner_stays_learner_on_membership_applied() { context.membership = Arc::new(mock_membership); let mut state = LearnerState::::new(3, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = state.handle_membership_applied(&context, &role_tx).await; + let result = state.handle_membership_applied(&context, &internal_event_tx).await; assert!(result.is_ok(), "MembershipApplied should succeed"); - let timeout_result = - tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()).await; + let timeout_result = tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await; if let Ok(Some(event)) = timeout_result { panic!("Should not send role transition when still Learner, got: {event:?}"); @@ -1142,7 +1175,7 @@ async fn test_learner_stays_learner_on_membership_applied() { /// /// Expected: /// - No role transition event -/// - handle_raft_event returns Ok() +/// - handle_inbound_event returns Ok() /// /// Original: test_learner_node_not_found_on_membership_applied #[tokio::test] @@ -1161,16 +1194,19 @@ async fn test_learner_node_not_found_on_membership_applied() { context.membership = Arc::new(mock_membership); let mut state = LearnerState::::new(3, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel(); - let result = state.handle_membership_applied(&context, &role_tx).await; + let result = state.handle_membership_applied(&context, &internal_event_tx).await; assert!( result.is_ok(), "MembershipApplied should succeed even when node not found" ); - let timeout_result = - tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()).await; + let timeout_result = tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await; if let Ok(Some(event)) = timeout_result { panic!("Should not send role transition when node not found, got: {event:?}"); @@ -1198,18 +1234,18 @@ mod snapshot_tests { let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); assert!( state.handle_log_purge_completed(LogId { term: 1, index: 1 }).is_ok(), "Stale LogPurgeCompleted should be silently ignored" ); assert!( - state.handle_promote_ready_learners(&context, &role_tx).await.is_ok(), + state.handle_promote_ready_learners(&context, &internal_event_tx).await.is_ok(), "Stale PromoteReadyLearners should be silently ignored" ); assert!( - state.handle_self_removed(&role_tx).is_ok(), + state.handle_self_removed(&internal_event_tx).is_ok(), "Stale StepDownSelfRemoved should be silently ignored" ); } @@ -1227,10 +1263,10 @@ mod snapshot_tests { let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = LearnerState::::new(1, context.node_config.clone()); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); // First trigger — starts background snapshot - let result1 = state.handle_create_snapshot(&context, &role_tx).await; + let result1 = state.handle_create_snapshot(&context, &internal_event_tx).await; assert!(result1.is_ok(), "First CreateSnapshotEvent should succeed"); assert!( state.snapshot_in_progress.load(Ordering::SeqCst), @@ -1238,7 +1274,7 @@ mod snapshot_tests { ); // Second trigger while first is still running — must be a no-op - let result2 = state.handle_create_snapshot(&context, &role_tx).await; + let result2 = state.handle_create_snapshot(&context, &internal_event_tx).await; assert!( result2.is_ok(), "Second CreateSnapshotEvent should return Ok (ignored)" @@ -1278,8 +1314,10 @@ mod snapshot_tests { }; let snapshot_result = Ok((metadata, std::path::PathBuf::from("/tmp/test_snapshot.bin"))); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let result = state + .handle_snapshot_created(snapshot_result, &context, &internal_event_tx) + .await; assert!(result.is_ok(), "SnapshotCreated should succeed"); assert!( @@ -1312,8 +1350,10 @@ mod snapshot_tests { let snapshot_result = Err(crate::Error::Fatal("Snapshot creation failed".to_string())); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); - let result = state.handle_snapshot_created(snapshot_result, &context, &role_tx).await; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let result = state + .handle_snapshot_created(snapshot_result, &context, &internal_event_tx) + .await; assert!( result.is_ok(), @@ -1344,10 +1384,10 @@ mod snapshot_tests { /// - FatalError event from StateMachine component /// /// # When -/// - Learner handles FatalError event via handle_raft_event() +/// - Learner handles FatalError event via handle_inbound_event() /// /// # Then -/// - handle_raft_event() returns Error::Fatal +/// - handle_inbound_event() returns Error::Fatal /// - Error message contains source and error details /// - No role transition events are sent (log replication is aborted) #[tokio::test] @@ -1362,21 +1402,21 @@ async fn test_learner_handles_fatal_error_returns_error() { let mut learner = LearnerState::::new(1, context.node_config.clone()); // Create FatalError event - let fatal_error = RaftEvent::FatalError { + let fatal_error = InboundEvent::FatalError { source: "StateMachine".to_string(), error: "Disk failure".to_string(), }; - // Create role event channel - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + // Create internal event channel + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // Handle the FatalError event - let result = learner.handle_raft_event(fatal_error, &context, role_tx).await; + let result = learner.handle_inbound_event(fatal_error, &context, internal_event_tx).await; - // VERIFY 1: handle_raft_event() returns Error::Fatal + // VERIFY 1: handle_inbound_event() returns Error::Fatal assert!( result.is_err(), - "Expected handle_raft_event to return Err, got: {result:?}" + "Expected handle_inbound_event to return Err, got: {result:?}" ); // VERIFY 2: Error is Fatal and contains source information @@ -1390,9 +1430,9 @@ async fn test_learner_handles_fatal_error_returns_error() { other => panic!("Expected Error::Fatal, got: {other:?}"), } - // VERIFY 3: No role events sent + // VERIFY 3: No internal events sent assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "No role transition events should be sent during FatalError handling" ); } @@ -1496,7 +1536,7 @@ async fn test_learner_serves_eventual_read_locally() { /// - State machine handler indicates snapshot should be taken /// /// Expected: -/// - RoleEvent::CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) +/// - InternalEvent::CreateSnapshotEvent is sent directly on internal_event_tx (P2 unbounded) /// - No ReprocessEvent wrapper — direct send eliminates the bounded event_tx deadlock path #[tokio::test] async fn test_apply_completed_triggers_snapshot_when_condition_met() { @@ -1521,10 +1561,10 @@ async fn test_apply_completed_triggers_snapshot_when_condition_met() { let mut learner = LearnerState::::new(1, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // ACTION: Handle ApplyCompleted event - let result = learner.handle_apply_completed(100, vec![], &context, &role_tx).await; + let result = learner.handle_apply_completed(100, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds assert!( @@ -1532,16 +1572,16 @@ async fn test_apply_completed_triggers_snapshot_when_condition_met() { "ApplyCompleted should be handled successfully, got: {result:?}" ); - // VERIFY 2: CreateSnapshotEvent is sent directly on role_tx (P2 unbounded) - let event = role_rx.try_recv().expect("Should receive snapshot trigger event"); + // VERIFY 2: CreateSnapshotEvent is sent directly on internal_event_tx (P2 unbounded) + let event = internal_event_rx.try_recv().expect("Should receive snapshot trigger event"); assert!( - matches!(event, RoleEvent::CreateSnapshotEvent), - "Expected RoleEvent::CreateSnapshotEvent, got: {event:?}" + matches!(event, InternalEvent::CreateSnapshotEvent), + "Expected InternalEvent::CreateSnapshotEvent, got: {event:?}" ); // VERIFY 3: No additional events queued assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should only send one snapshot event" ); } @@ -1581,10 +1621,10 @@ async fn test_apply_completed_does_not_trigger_snapshot_when_condition_not_met() let mut learner = LearnerState::::new(1, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // ACTION: Handle ApplyCompleted event - let result = learner.handle_apply_completed(50, vec![], &context, &role_tx).await; + let result = learner.handle_apply_completed(50, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds assert!( @@ -1594,7 +1634,7 @@ async fn test_apply_completed_does_not_trigger_snapshot_when_condition_not_met() // VERIFY 2: No snapshot event is sent assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should not send snapshot event when condition is not met" ); } @@ -1629,10 +1669,10 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { let mut learner = LearnerState::::new(1, context.node_config.clone()); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); // ACTION: Handle ApplyCompleted event - let result = learner.handle_apply_completed(100, vec![], &context, &role_tx).await; + let result = learner.handle_apply_completed(100, vec![], &context, &internal_event_tx).await; // VERIFY 1: Event handling succeeds assert!( @@ -1642,7 +1682,7 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { // VERIFY 2: No snapshot event is sent (snapshot disabled in config) assert!( - role_rx.try_recv().is_err(), + internal_event_rx.try_recv().is_err(), "Should not send snapshot event when snapshot is disabled in config" ); } @@ -1689,10 +1729,15 @@ async fn test_learner_acks_immediately_after_memory_write() { leader_commit_index: 0, }; let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let raft_event = RaftEvent::AppendEntries(append_request, vec![resp_tx]); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - assert!(state.handle_raft_event(raft_event, &context, role_tx).await.is_ok()); + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); // MemFirst: ACK sent immediately let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); @@ -1740,16 +1785,16 @@ async fn test_learner_install_snapshot_reports_success_after_all_chunks_applied( state.update_current_term(2); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (tx, rx) = mpsc::channel(32); tx.send(SnapshotChunk::default()).await.unwrap(); drop(tx); state - .handle_raft_event( - RaftEvent::InstallSnapshotChunk(rx, resp_tx), + .handle_inbound_event( + InboundEvent::InstallSnapshotChunk(rx, resp_tx), &context, - role_tx, + internal_event_tx, ) .await .unwrap(); @@ -1814,17 +1859,17 @@ async fn test_learner_install_snapshot_does_not_report_success_on_mid_chunk_fail state.update_current_term(2); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (tx, rx) = mpsc::channel(32); tx.send(SnapshotChunk::default()).await.unwrap(); drop(tx); - // handle_raft_event returns Err (apply failed) — that is expected + // handle_inbound_event returns Err (apply failed) — that is expected let _ = state - .handle_raft_event( - RaftEvent::InstallSnapshotChunk(rx, resp_tx), + .handle_inbound_event( + InboundEvent::InstallSnapshotChunk(rx, resp_tx), &context, - role_tx, + internal_event_tx, ) .await; @@ -1885,17 +1930,17 @@ async fn test_learner_install_snapshot_reports_failure_when_apply_fails_after_tr state.update_current_term(2); let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let (role_tx, _role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); let (tx, rx) = mpsc::channel(32); tx.send(SnapshotChunk::default()).await.unwrap(); drop(tx); // Learner propagates apply errors — ignore the return value here let _ = state - .handle_raft_event( - RaftEvent::InstallSnapshotChunk(rx, resp_tx), + .handle_inbound_event( + InboundEvent::InstallSnapshotChunk(rx, resp_tx), &context, - role_tx, + internal_event_tx, ) .await; diff --git a/d-engine-core/src/raft_role/mod.rs b/d-engine-core/src/raft_role/mod.rs index 8e8f3a9f..cc8adb32 100644 --- a/d-engine-core/src/raft_role/mod.rs +++ b/d-engine-core/src/raft_role/mod.rs @@ -44,9 +44,9 @@ use tokio::time::Instant; use tracing::debug; use tracing::trace; +use super::InboundEvent; +use super::InternalEvent; use super::RaftContext; -use super::RaftEvent; -use super::RoleEvent; use crate::Result; use crate::TypeConfig; @@ -381,13 +381,13 @@ impl RaftRole { pub(crate) async fn handle_zombie_detected( &mut self, node_id: u32, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ctx: &RaftContext, ) -> Result<()> where T: TypeConfig, { - self.state_mut().handle_zombie_detected(node_id, role_tx, ctx).await + self.state_mut().handle_zombie_detected(node_id, internal_event_tx, ctx).await } pub(crate) fn handle_snapshot_push_completed( @@ -403,27 +403,29 @@ impl RaftRole { pub(crate) async fn tick( &mut self, - role_tx: &mpsc::UnboundedSender, - event_tx: &mpsc::Sender, + internal_event_tx: &mpsc::UnboundedSender, + event_tx: &mpsc::Sender, ctx: &RaftContext, ) -> Result<()> where T: TypeConfig, { trace!("raft_role:tick"); - self.state_mut().tick(role_tx, event_tx, ctx).await + self.state_mut().tick(internal_event_tx, event_tx, ctx).await } - pub(crate) async fn handle_raft_event( + pub(crate) async fn handle_inbound_event( &mut self, - raft_event: RaftEvent, + inbound_event: InboundEvent, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, ) -> Result<()> where T: TypeConfig, { - self.state_mut().handle_raft_event(raft_event, ctx, role_tx).await + self.state_mut() + .handle_inbound_event(inbound_event, ctx, internal_event_tx) + .await } /// Fire-and-forget noop to confirm quorum; delegates to LeaderState only. @@ -431,10 +433,10 @@ impl RaftRole { pub(crate) async fn initiate_noop_commit( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { if let RaftRole::Leader(state) = self { - state.initiate_noop_commit(ctx, role_tx).await + state.initiate_noop_commit(ctx, internal_event_tx).await } else { Ok(()) } @@ -463,9 +465,9 @@ impl RaftRole { pub(crate) async fn flush_cmd_buffers( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - self.state_mut().flush_cmd_buffers(ctx, role_tx).await + self.state_mut().flush_cmd_buffers(ctx, internal_event_tx).await } /// Dispatch ApplyCompleted to the current role state. @@ -474,12 +476,14 @@ impl RaftRole { last_index: u64, results: Vec, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> crate::Result<()> where T: TypeConfig, { - self.state_mut().handle_apply_completed(last_index, results, ctx, role_tx).await + self.state_mut() + .handle_apply_completed(last_index, results, ctx, internal_event_tx) + .await } /// Dispatch LogFlushed(durable) to the current role state. @@ -488,9 +492,9 @@ impl RaftRole { &mut self, durable: u64, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) { - self.state_mut().handle_log_flushed(durable, ctx, role_tx).await + self.state_mut().handle_log_flushed(durable, ctx, internal_event_tx).await } /// Dispatch AppendResult from a per-follower worker back to the current role state. @@ -499,9 +503,11 @@ impl RaftRole { follower_id: u32, result: crate::Result, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> crate::Result<()> { - self.state_mut().handle_append_result(follower_id, result, ctx, role_tx).await + self.state_mut() + .handle_append_result(follower_id, result, ctx, internal_event_tx) + .await } /// Trigger an independent snapshot on this role (Raft §7 — each server snapshots @@ -510,9 +516,9 @@ impl RaftRole { pub(crate) async fn handle_create_snapshot( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - self.state_mut().handle_create_snapshot(ctx, role_tx).await + self.state_mut().handle_create_snapshot(ctx, internal_event_tx).await } /// Process the completed snapshot result (success or error). @@ -523,9 +529,9 @@ impl RaftRole { &mut self, result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - self.state_mut().handle_snapshot_created(result, ctx, role_tx).await + self.state_mut().handle_snapshot_created(result, ctx, internal_event_tx).await } /// Advance the purge boundary after log entries up to `purged_id` are removed. @@ -544,9 +550,9 @@ impl RaftRole { pub(crate) async fn handle_promote_ready_learners( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - self.state_mut().handle_promote_ready_learners(ctx, role_tx).await + self.state_mut().handle_promote_ready_learners(ctx, internal_event_tx).await } /// Node was removed from cluster membership; step down immediately per Raft protocol. @@ -554,9 +560,9 @@ impl RaftRole { /// self-removal via config change). pub(crate) fn handle_self_removed( &mut self, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - self.state_mut().handle_self_removed(role_tx) + self.state_mut().handle_self_removed(internal_event_tx) } /// Membership config change applied to state — refresh any role-local derived state. @@ -566,9 +572,9 @@ impl RaftRole { pub(crate) async fn handle_membership_applied( &mut self, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { - self.state_mut().handle_membership_applied(ctx, role_tx).await + self.state_mut().handle_membership_applied(ctx, internal_event_tx).await } } diff --git a/d-engine-core/src/raft_role/raft_role_test.rs b/d-engine-core/src/raft_role/raft_role_test.rs index 222d6d1d..155f75a6 100644 --- a/d-engine-core/src/raft_role/raft_role_test.rs +++ b/d-engine-core/src/raft_role/raft_role_test.rs @@ -697,7 +697,7 @@ fn test_update_voted_for_learner_discovers_leader() { // Only the leader can propose membership changes (BatchRemove). When a // ZombieDetected signal arrives while the node is a Follower, Candidate, or // Learner the call must return Ok(()) immediately without emitting any event on -// role_tx. Recovery from the lost signal is handled by the health-monitor +// internal_event_tx. Recovery from the lost signal is handled by the health-monitor // counter-reset mechanism (see health_monitor_test). // ============================================================================ @@ -721,14 +721,14 @@ async fn test_handle_zombie_detected_is_noop_on_follower() { let cfg = make_test_config(); let mut role = RaftRole::Follower(Box::new(FollowerState::new(1, cfg, None, None))); let (ctx, _shutdown_tx) = make_test_context(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); - let result = role.handle_zombie_detected(42, &role_tx, &ctx).await; + let result = role.handle_zombie_detected(42, &internal_event_tx, &ctx).await; assert!(result.is_ok(), "Follower must not error on ZombieDetected"); assert!( - role_rx.try_recv().is_err(), - "Follower must not emit any RoleEvent for ZombieDetected" + internal_event_rx.try_recv().is_err(), + "Follower must not emit any InternalEvent for ZombieDetected" ); } @@ -737,14 +737,14 @@ async fn test_handle_zombie_detected_is_noop_on_candidate() { let cfg = make_test_config(); let mut role = RaftRole::Candidate(Box::new(CandidateState::new(1, cfg))); let (ctx, _shutdown_tx) = make_test_context(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); - let result = role.handle_zombie_detected(42, &role_tx, &ctx).await; + let result = role.handle_zombie_detected(42, &internal_event_tx, &ctx).await; assert!(result.is_ok(), "Candidate must not error on ZombieDetected"); assert!( - role_rx.try_recv().is_err(), - "Candidate must not emit any RoleEvent for ZombieDetected" + internal_event_rx.try_recv().is_err(), + "Candidate must not emit any InternalEvent for ZombieDetected" ); } @@ -753,13 +753,13 @@ async fn test_handle_zombie_detected_is_noop_on_learner() { let cfg = make_test_config(); let mut role = RaftRole::Learner(Box::new(LearnerState::new(1, cfg))); let (ctx, _shutdown_tx) = make_test_context(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); - let result = role.handle_zombie_detected(42, &role_tx, &ctx).await; + let result = role.handle_zombie_detected(42, &internal_event_tx, &ctx).await; assert!(result.is_ok(), "Learner must not error on ZombieDetected"); assert!( - role_rx.try_recv().is_err(), - "Learner must not emit any RoleEvent for ZombieDetected" + internal_event_rx.try_recv().is_err(), + "Learner must not emit any InternalEvent for ZombieDetected" ); } diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index 67601446..3a6b7d46 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -17,17 +17,17 @@ use super::RaftRole; use super::SharedState; use super::StateSnapshot; use crate::AppendResponseWithUpdates; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MaybeCloneOneshotSender; use crate::Membership; use crate::MembershipError; use crate::NetworkError; use crate::NewCommitData; use crate::RaftContext; -use crate::RaftEvent; use crate::RaftLog; use crate::ReplicationCore; use crate::Result; -use crate::RoleEvent; use crate::StateMachineHandler; use crate::StateTransitionError; use crate::TypeConfig; @@ -93,7 +93,7 @@ pub trait RaftRoleState: Send + Sync + 'static { async fn handle_zombie_detected( &mut self, _node_id: u32, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, _ctx: &RaftContext, ) -> Result<()> { // Default: no-op for non-leader roles @@ -213,7 +213,7 @@ pub trait RaftRoleState: Send + Sync + 'static { role: i32, current_term: u64, new_commit_index: u64, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { if let Err(e) = self.update_commit_index(new_commit_index) { error!("Follower::update_commit_index: {:?}", e); @@ -225,8 +225,8 @@ pub trait RaftRoleState: Send + Sync + 'static { new_commit_index ); - role_tx - .send(RoleEvent::NotifyNewCommitIndex(NewCommitData { + internal_event_tx + .send(InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index, role, current_term, @@ -260,16 +260,16 @@ pub trait RaftRoleState: Send + Sync + 'static { async fn tick( &mut self, - role_tx: &mpsc::UnboundedSender, - event_tx: &mpsc::Sender, + internal_event_tx: &mpsc::UnboundedSender, + event_tx: &mpsc::Sender, ctx: &RaftContext, ) -> Result<()>; - async fn handle_raft_event( + async fn handle_inbound_event( &mut self, - raft_event: RaftEvent, + inbound_event: InboundEvent, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, ) -> Result<()>; /// Push single client command directly to role's internal buffer (zero-copy). @@ -361,7 +361,7 @@ pub trait RaftRoleState: Send + Sync + 'static { async fn flush_cmd_buffers( &mut self, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Default implementation for non-leader: nothing to flush Ok(()) @@ -376,7 +376,7 @@ pub trait RaftRoleState: Send + Sync + 'static { _last_index: u64, _results: Vec, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Ok(()) } @@ -388,7 +388,7 @@ pub trait RaftRoleState: Send + Sync + 'static { &mut self, _durable: u64, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) { // Candidate: no-op } @@ -401,7 +401,7 @@ pub trait RaftRoleState: Send + Sync + 'static { _follower_id: u32, _result: crate::Result, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> crate::Result<()> { // Non-leader: stale result arrived after step-down — ignore safely. Ok(()) @@ -453,12 +453,12 @@ pub trait RaftRoleState: Send + Sync + 'static { append_entries_request: AppendEntriesRequest, senders: Vec>>, ctx: &RaftContext, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, state_snapshot: &StateSnapshot, ) -> Result<()> { let _timer = ScopedTimer::new("handle_append_entries_request_workflow"); debug!( - "handle_raft_event::RaftEvent::AppendEntries: {:?}", + "handle_inbound_event::InboundEvent::AppendEntries: {:?}", &append_entries_request ); self.reset_timer(); @@ -504,12 +504,12 @@ pub trait RaftRoleState: Send + Sync + 'static { // Event-driven: avoids redundant notifications on every heartbeat // Performance: ~9ns check overhead, saves ~100ns redundant sends if is_new_leader { - role_tx.send(RoleEvent::LeaderDiscovered(new_leader_id, request_term)).map_err( - |e| { + internal_event_tx + .send(InternalEvent::LeaderDiscovered(new_leader_id, request_term)) + .map_err(|e| { error!("Failed to send LeaderDiscovered: {e:?}"); NetworkError::SingalSendFailed(format!("{:?}", e)) - }, - )?; + })?; } if my_term < request_term { @@ -534,7 +534,7 @@ pub trait RaftRoleState: Send + Sync + 'static { state_snapshot.role, state_snapshot.current_term, commit, - &role_tx, + &internal_event_tx, ) { error!( @@ -576,7 +576,7 @@ pub trait RaftRoleState: Send + Sync + 'static { } } - error("handle_raft_event", &e); + error("handle_inbound_event", &e); return Err(e); } } @@ -594,7 +594,7 @@ pub trait RaftRoleState: Send + Sync + 'static { async fn handle_create_snapshot( &mut self, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Err(ConsensusError::RoleViolation { current_role: "Candidate", @@ -612,7 +612,7 @@ pub trait RaftRoleState: Send + Sync + 'static { &mut self, _result: crate::Result<(SnapshotMetadata, std::path::PathBuf)>, _ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Err(ConsensusError::RoleViolation { current_role: "Candidate", @@ -643,13 +643,13 @@ pub trait RaftRoleState: Send + Sync + 'static { async fn handle_promote_ready_learners( &mut self, ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { Err(ConsensusError::RoleViolation { current_role: "None Leader", required_role: "Leader", context: format!( - "None Leader node {} receives RaftEvent::PromoteReadyLearners", + "None Leader node {} receives InboundEvent::PromoteReadyLearners", ctx.node_id ), } @@ -661,7 +661,7 @@ pub trait RaftRoleState: Send + Sync + 'static { /// Default: warn + Ok(()) (defensive; only Leader should receive this). fn handle_self_removed( &mut self, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { warn!( "Node {} received StepDownSelfRemoved in non-leader role — ignoring", @@ -677,7 +677,7 @@ pub trait RaftRoleState: Send + Sync + 'static { async fn handle_membership_applied( &mut self, ctx: &RaftContext, - _role_tx: &mpsc::UnboundedSender, + _internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { // Followers don't maintain cluster metadata cache // This event is only relevant for leaders @@ -686,7 +686,7 @@ pub trait RaftRoleState: Send + Sync + 'static { current_role: "None Leader/Learner", required_role: "Leader/Learner", context: format!( - "None Leader/Learner node {} receives RaftEvent::PromoteReadyLearners", + "None Leader/Learner node {} receives InboundEvent::PromoteReadyLearners", ctx.node_id ), } @@ -694,15 +694,17 @@ pub trait RaftRoleState: Send + Sync + 'static { } } -/// Send a RaftEvent back into the role event loop for reprocessing. -pub(super) fn send_replay_raft_event( - role_tx: &mpsc::UnboundedSender, - raft_event: RaftEvent, +/// Send a InboundEvent back into the internal event loop for reprocessing. +pub(super) fn send_replay_inbound_event( + internal_event_tx: &mpsc::UnboundedSender, + inbound_event: InboundEvent, ) -> Result<()> { - role_tx.send(RoleEvent::ReprocessEvent(Box::new(raft_event))).map_err(|e| { - error!("Failed to send: {e:?}"); - NetworkError::SingalSendFailed(format!("{:?}", e)).into() - }) + internal_event_tx + .send(InternalEvent::ReprocessEvent(Box::new(inbound_event))) + .map_err(|e| { + error!("Failed to send: {e:?}"); + NetworkError::SingalSendFailed(format!("{:?}", e)).into() + }) } /// Check snapshot condition and trigger if met. Used by all role states after SM apply. @@ -714,7 +716,7 @@ pub(super) fn check_and_trigger_snapshot( role: i32, current_term: u64, ctx: &RaftContext, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, ) -> Result<()> { if ctx.node_config.raft.snapshot.enable && ctx.state_machine_handler().should_snapshot(NewCommitData { @@ -723,7 +725,7 @@ pub(super) fn check_and_trigger_snapshot( current_term, }) { - role_tx.send(RoleEvent::CreateSnapshotEvent).map_err(|e| { + internal_event_tx.send(InternalEvent::CreateSnapshotEvent).map_err(|e| { error!("Failed to send: {e:?}"); NetworkError::SingalSendFailed(format!("{:?}", e)) })?; diff --git a/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs b/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs index 71037e93..116b5153 100644 --- a/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs +++ b/d-engine-core/src/raft_test/drain_based_batch_architecture_tests.rs @@ -187,10 +187,10 @@ async fn test_p0_leader_single_command_drain_immediately() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Leader - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(crate::RoleEvent::BecomeLeader) + raft.handle_internal_event(crate::InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -219,9 +219,9 @@ async fn test_p0_leader_single_command_drain_immediately() { leader.push_client_cmd(cmd, &raft.ctx); // **Step 3**: Flush buffers (simulate drain) - let (role_tx, _role_rx) = tokio::sync::mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); leader - .flush_cmd_buffers(&raft.ctx, &role_tx) + .flush_cmd_buffers(&raft.ctx, &internal_event_tx) .await .expect("flush should succeed"); @@ -279,10 +279,10 @@ async fn test_p0_leader_high_load_max_batch_cap() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Leader - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(crate::RoleEvent::BecomeLeader) + raft.handle_internal_event(crate::InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -296,7 +296,7 @@ async fn test_p0_leader_high_load_max_batch_cap() { const MAX_BATCH_SIZE: usize = 100; const ROUNDS: usize = 10; - let (role_tx, _role_rx) = tokio::sync::mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); if let RaftRole::Leader(ref mut leader) = raft.role { for round in 0..ROUNDS { @@ -322,7 +322,7 @@ async fn test_p0_leader_high_load_max_batch_cap() { // Flush: sends exactly the pushed batch to replication leader - .flush_cmd_buffers(&raft.ctx, &role_tx) + .flush_cmd_buffers(&raft.ctx, &internal_event_tx) .await .expect("flush should succeed"); } @@ -554,7 +554,7 @@ async fn test_p0_candidate_commands_handling() { raft.ctx.handlers.state_machine_handler = Arc::new(state_machine_handler); // Transition to Candidate - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); @@ -726,10 +726,10 @@ async fn test_p1_medium_load_natural_batching() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Leader - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(crate::RoleEvent::BecomeLeader) + raft.handle_internal_event(crate::InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -763,9 +763,9 @@ async fn test_p1_medium_load_natural_batching() { } // **Step 2**: Single flush drains all 50 commands in one batch - let (role_tx, _role_rx) = tokio::sync::mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); leader - .flush_cmd_buffers(&raft.ctx, &role_tx) + .flush_cmd_buffers(&raft.ctx, &internal_event_tx) .await .expect("flush should succeed"); @@ -840,10 +840,10 @@ async fn test_p1_mixed_workload_read_write_coexistence() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Leader - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(crate::RoleEvent::BecomeLeader) + raft.handle_internal_event(crate::InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -894,9 +894,9 @@ async fn test_p1_mixed_workload_read_write_coexistence() { } // **Step 2**: Flush all commands - let (role_tx, _role_rx) = tokio::sync::mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); leader - .flush_cmd_buffers(&raft.ctx, &role_tx) + .flush_cmd_buffers(&raft.ctx, &internal_event_tx) .await .expect("flush should succeed"); @@ -948,10 +948,10 @@ async fn test_p1_burst_load_recovery_after_spike() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to Leader - raft.handle_role_event(crate::RoleEvent::BecomeCandidate) + raft.handle_internal_event(crate::InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(crate::RoleEvent::BecomeLeader) + raft.handle_internal_event(crate::InternalEvent::BecomeLeader) .await .expect("Should become Leader"); @@ -978,11 +978,11 @@ async fn test_p1_burst_load_recovery_after_spike() { } // **Step 2**: Flush burst (multiple cycles expected) - let (role_tx, _role_rx) = tokio::sync::mpsc::unbounded_channel(); + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); let mut burst_flushes = 0; for _ in 0..20 { leader - .flush_cmd_buffers(&raft.ctx, &role_tx) + .flush_cmd_buffers(&raft.ctx, &internal_event_tx) .await .expect("flush should succeed"); burst_flushes += 1; @@ -1011,7 +1011,7 @@ async fn test_p1_burst_load_recovery_after_spike() { // Flush after each trickle command leader - .flush_cmd_buffers(&raft.ctx, &role_tx) + .flush_cmd_buffers(&raft.ctx, &internal_event_tx) .await .expect("flush should succeed"); diff --git a/d-engine-core/src/raft_test/leader_discovered_tests.rs b/d-engine-core/src/raft_test/leader_discovered_tests.rs index c5eda850..db6a9d40 100644 --- a/d-engine-core/src/raft_test/leader_discovered_tests.rs +++ b/d-engine-core/src/raft_test/leader_discovered_tests.rs @@ -1,4 +1,4 @@ -//! # RoleEvent Handling Tests - Leader Discovery Category +//! # InternalEvent Handling Tests - Leader Discovery Category //! //! **Original location**: raft_test.rs L379-514 //! **Test count**: 5 tests @@ -11,13 +11,13 @@ //! - LeaderDiscovered event triggers notifications but no state change //! - Multiple listeners receive the event //! - Event deduplication and watch channel behavior -//! - RoleEvent::LeaderDiscovered creation and serialization +//! - InternalEvent::LeaderDiscovered creation and serialization //! use tokio::sync::watch; +use crate::InternalEvent; use crate::Raft; -use crate::RoleEvent; use crate::test_utils::MockBuilder; use crate::test_utils::MockTypeConfig; @@ -34,7 +34,7 @@ async fn test_leader_discovered_event_handling() { // Send LeaderDiscovered event let leader_id = 3; let term = 5; - raft.handle_role_event(RoleEvent::LeaderDiscovered(leader_id, term)) + raft.handle_internal_event(InternalEvent::LeaderDiscovered(leader_id, term)) .await .expect("Should handle LeaderDiscovered"); @@ -56,7 +56,7 @@ async fn test_leader_discovered_no_state_change() { let initial_role = raft.role.as_i32(); // Send LeaderDiscovered event - raft.handle_role_event(RoleEvent::LeaderDiscovered(3, 5)) + raft.handle_internal_event(InternalEvent::LeaderDiscovered(3, 5)) .await .expect("Should handle LeaderDiscovered"); @@ -81,7 +81,7 @@ async fn test_leader_discovered_multiple_listeners() { // Send LeaderDiscovered event let leader_id = 2; let term = 10; - raft.handle_role_event(RoleEvent::LeaderDiscovered(leader_id, term)) + raft.handle_internal_event(InternalEvent::LeaderDiscovered(leader_id, term)) .await .expect("Should handle LeaderDiscovered"); @@ -108,10 +108,10 @@ async fn test_leader_discovered_with_deduplication() { raft.register_leader_change_listener(leader_tx); // Send same leader multiple times - raft.handle_role_event(RoleEvent::LeaderDiscovered(2, 5)) + raft.handle_internal_event(InternalEvent::LeaderDiscovered(2, 5)) .await .expect("Should handle first"); - raft.handle_role_event(RoleEvent::LeaderDiscovered(2, 5)) + raft.handle_internal_event(InternalEvent::LeaderDiscovered(2, 5)) .await .expect("Should handle second (duplicate)"); @@ -133,15 +133,15 @@ async fn test_leader_discovered_with_deduplication() { } #[test] -fn test_role_event_leader_discovered_creation() { +fn test_internal_event_leader_discovered_creation() { // Test creating LeaderDiscovered event let leader_id = 5; let term = 20; - let event = RoleEvent::LeaderDiscovered(leader_id, term); + let event = InternalEvent::LeaderDiscovered(leader_id, term); // Verify we can match on it match event { - RoleEvent::LeaderDiscovered(id, t) => { + InternalEvent::LeaderDiscovered(id, t) => { assert_eq!(id, leader_id); assert_eq!(t, term); } diff --git a/d-engine-core/src/raft_test/merge_append_entries_tests.rs b/d-engine-core/src/raft_test/merge_append_entries_tests.rs index da63adf0..bbea3e38 100644 --- a/d-engine-core/src/raft_test/merge_append_entries_tests.rs +++ b/d-engine-core/src/raft_test/merge_append_entries_tests.rs @@ -1,7 +1,7 @@ //! # AppendEntries Merge Tests //! //! Tests for `Raft::merge_append_entries()` — the function that coalesces -//! consecutive contiguous `RaftEvent::AppendEntries` events in the buffer +//! consecutive contiguous `InboundEvent::AppendEntries` events in the buffer //! into a single event before dispatch. //! //! ## Design decisions under test @@ -14,12 +14,12 @@ //! //! ## Setup pattern //! Each test builds a `Raft` via `MockBuilder`, pushes events -//! directly into `raft.buffered_raft_event`, calls `raft.merge_append_entries()`, +//! directly into `raft.buffered_inbound_event`, calls `raft.merge_append_entries()`, //! then inspects the resulting buffer. use crate::maybe_clone_oneshot::MaybeCloneOneshot; use crate::test_utils::MockBuilder; -use crate::{MaybeCloneOneshotReceiver, RaftEvent, RaftNodeConfig, RaftOneshot, mock_entries}; +use crate::{InboundEvent, MaybeCloneOneshotReceiver, RaftNodeConfig, RaftOneshot, mock_entries}; use d_engine_proto::common::LogId; use d_engine_proto::server::election::{VoteRequest, VoteResponse}; use d_engine_proto::server::replication::{AppendEntriesRequest, AppendEntriesResponse}; @@ -50,25 +50,25 @@ fn make_request( } } -/// Wrap a request and a fresh oneshot sender into `RaftEvent::AppendEntries`. +/// Wrap a request and a fresh oneshot sender into `InboundEvent::AppendEntries`. /// Returns `(event, receiver)` — the receiver lets tests verify send results. fn make_event( req: AppendEntriesRequest ) -> ( - RaftEvent, + InboundEvent, MaybeCloneOneshotReceiver>, ) { let (tx, rx) = MaybeCloneOneshot::new(); - (RaftEvent::AppendEntries(req, vec![tx]), rx) + (InboundEvent::AppendEntries(req, vec![tx]), rx) } fn make_none_append_entries_event() -> ( - RaftEvent, + InboundEvent, MaybeCloneOneshotReceiver>, ) { let (tx, rx) = MaybeCloneOneshot::new(); ( - RaftEvent::ReceiveVoteRequest( + InboundEvent::ReceiveVoteRequest( VoteRequest { term: 1, candidate_id: 2, @@ -96,10 +96,10 @@ fn make_none_append_entries_event() -> ( fn test_empty_buffer_is_noop() { let (_graceful_tx, graceful_rx) = watch::channel(()); let mut raft = MockBuilder::new(graceful_rx).build_raft(); - raft.buffered_raft_event = VecDeque::new(); + raft.buffered_inbound_event = VecDeque::new(); raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 0); + assert_eq!(raft.buffered_inbound_event.len(), 0); } /// A buffer with exactly one AppendEntries event is returned unchanged. @@ -122,19 +122,19 @@ fn test_single_append_entries_event_returned_unchanged() { let (event, _receiver) = make_event(request); queue.push_back(event); - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + assert_eq!(raft.buffered_inbound_event.len(), 1); + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.term, 5); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -163,29 +163,29 @@ fn test_first_non_append_entries_event_is_noop() { queue.push_back(event); } } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 2); + assert_eq!(raft.buffered_inbound_event.len(), 2); // VoteRequest - if let Some(RaftEvent::ReceiveVoteRequest(request, _senders)) = - raft.buffered_raft_event.pop_front() + if let Some(InboundEvent::ReceiveVoteRequest(request, _senders)) = + raft.buffered_inbound_event.pop_front() { assert_eq!(request.last_log_index, 11); } else { - panic!("expected RaftEvent::ReceiveVoteRequest, got something else"); + panic!("expected InboundEvent::ReceiveVoteRequest, got something else"); } // AE - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -216,19 +216,19 @@ fn test_merge_three_contiguous_entries_same_term() { queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + assert_eq!(raft.buffered_inbound_event.len(), 1); + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 15); assert_eq!(senders.len(), 3); assert_eq!(request.term, 5); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -256,14 +256,14 @@ fn test_merge_preserves_first_request_metadata() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 10); assert_eq!(senders.len(), 2); @@ -271,7 +271,7 @@ fn test_merge_preserves_first_request_metadata() { assert_eq!(request.prev_log_index, 9); assert_eq!(request.leader_commit_index, 99); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -300,33 +300,33 @@ fn test_term_mismatch_stops_merge() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 2); + assert_eq!(raft.buffered_inbound_event.len(), 2); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 10); assert_eq!(senders.len(), 2); assert_eq!(request.term, 5); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } // back - match raft.buffered_raft_event.pop_back() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_back() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.term, 6); assert_eq!(request.prev_log_index, 19); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -354,33 +354,33 @@ fn test_non_contiguous_prev_log_index_stops_merge() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 2); + assert_eq!(raft.buffered_inbound_event.len(), 2); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.term, 5); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } // back - match raft.buffered_raft_event.pop_back() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_back() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.term, 5); assert_eq!(request.prev_log_index, 20); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -414,21 +414,21 @@ fn test_heartbeat_absorbed_into_merge() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 10); assert_eq!(senders.len(), 3); assert_eq!(request.prev_log_index, 9); assert_eq!(request.leader_commit_index, 15); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -456,21 +456,21 @@ fn test_heartbeat_commit_index_promoted_via_max() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 10); assert_eq!(senders.len(), 3); assert_eq!(request.prev_log_index, 9); assert_eq!(request.leader_commit_index, 99); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -509,40 +509,40 @@ fn test_non_append_entries_event_stops_merge_fifo_preserved() { queue.push_back(event); } } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 3); + assert_eq!(raft.buffered_inbound_event.len(), 3); // AE - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } // VoteRequest - if let Some(RaftEvent::ReceiveVoteRequest(request, _senders)) = - raft.buffered_raft_event.pop_front() + if let Some(InboundEvent::ReceiveVoteRequest(request, _senders)) = + raft.buffered_inbound_event.pop_front() { assert_eq!(request.last_log_index, 11); } else { - panic!("expected RaftEvent::ReceiveVoteRequest, got something else"); + panic!("expected InboundEvent::ReceiveVoteRequest, got something else"); } // AE - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 14); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -571,12 +571,13 @@ async fn test_all_senders_collected_in_merge_order() { receivers.push(rx); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); - let Some(RaftEvent::AppendEntries(_, senders)) = raft.buffered_raft_event.pop_front() else { + let Some(InboundEvent::AppendEntries(_, senders)) = raft.buffered_inbound_event.pop_front() + else { panic!("expected AppendEntries"); }; assert_eq!(senders.len(), 3); @@ -605,7 +606,7 @@ async fn test_all_senders_collected_in_merge_order() { } /// Dropping a receiver before merge does not prevent or corrupt the merge. -/// The merge is a structural operation on RaftEvent variants; receiver +/// The merge is a structural operation on InboundEvent variants; receiver /// validity is only checked at dispatch time (send()), not during merge. /// /// Buffer before: [AE(prev=9, [10..14], rx dropped), AE(prev=14, [15..19])] @@ -630,20 +631,20 @@ fn test_dropped_receiver_does_not_affect_merge() { queue.push_back(event); } } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 10); assert_eq!(senders.len(), 2); assert_eq!(request.prev_log_index, 9); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -673,21 +674,21 @@ fn test_heartbeat_at_sequence_end_absorbed() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 2); assert_eq!(request.prev_log_index, 9); assert_eq!(request.leader_commit_index, 50); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -715,19 +716,19 @@ fn test_consecutive_heartbeats_absorbed() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { assert_eq!(request.entries.len(), 10); // 5 + 0 + 0 + 5 assert_eq!(senders.len(), 4); assert_eq!(request.prev_log_index, 9); assert_eq!(request.leader_commit_index, 30); // max(10, 20, 30, 10) } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -752,21 +753,21 @@ fn test_all_heartbeats_merged_into_one() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 1); + assert_eq!(raft.buffered_inbound_event.len(), 1); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 0); assert_eq!(senders.len(), 2); assert_eq!(request.prev_log_index, 9); assert_eq!(request.leader_commit_index, 99); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -798,32 +799,32 @@ fn test_newer_term_first_older_term_second_stops_merge() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 2); + assert_eq!(raft.buffered_inbound_event.len(), 2); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 9); assert_eq!(request.term, 6); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } // back - match raft.buffered_raft_event.pop_back() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_back() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 14); assert_eq!(request.term, 5); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } @@ -857,31 +858,31 @@ fn test_max_merge_entries_limit_stops_merge() { let (event, _receiver) = make_event(request); queue.push_back(event); } - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; raft.merge_append_entries(); - assert_eq!(raft.buffered_raft_event.len(), 2); + assert_eq!(raft.buffered_inbound_event.len(), 2); // front - match raft.buffered_raft_event.pop_front() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_front() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 9); assert_eq!(request.term, 6); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } // back - match raft.buffered_raft_event.pop_back() { - Some(RaftEvent::AppendEntries(request, senders)) => { + match raft.buffered_inbound_event.pop_back() { + Some(InboundEvent::AppendEntries(request, senders)) => { let entries = request.entries; assert_eq!(entries.len(), 5); assert_eq!(senders.len(), 1); assert_eq!(request.prev_log_index, 14); assert_eq!(request.term, 6); } - _ => panic!("expected RaftEvent::AppendEntries, got something else"), + _ => panic!("expected InboundEvent::AppendEntries, got something else"), } } diff --git a/d-engine-core/src/raft_test/mod.rs b/d-engine-core/src/raft_test/mod.rs index 1012f627..3f983a82 100644 --- a/d-engine-core/src/raft_test/mod.rs +++ b/d-engine-core/src/raft_test/mod.rs @@ -58,9 +58,9 @@ //! - **A5.2**: Learner promotion to Follower //! - **A5.3**: Multiple learners in cluster (tracked separately from voters) //! -//! ### B. RoleEvent Handling Tests (Individual Event Processing) +//! ### B. InternalEvent Handling Tests (Individual Event Processing) //! -//! These tests verify that each RoleEvent variant is processed correctly and sends +//! These tests verify that each InternalEvent variant is processed correctly and sends //! appropriate notifications to listeners. //! //! #### B1. BecomeFollower Event @@ -98,23 +98,23 @@ //! #### B7. ReprocessEvent Event //! - **B7.1**: Event re-queued to event_rx for reprocessing //! -//! #### B8. CreateSnapshotEvent (migrated from P4 event_tx → P2 role_tx) +//! #### B8. CreateSnapshotEvent (migrated from P4 event_tx → P2 internal_event_tx) //! - **B8.1**: Follower — dispatch returns Ok, snapshot task spawned in background //! - **B8.2**: Candidate — RoleViolation is non-fatal, swallowed by dispatch layer //! -//! #### B9. SnapshotCreated (migrated from P4 event_tx → P2 role_tx) +//! #### B9. SnapshotCreated (migrated from P4 event_tx → P2 internal_event_tx) //! - **B9.1**: Follower + Err payload — dispatch returns Ok (non-fatal error swallowed) //! -//! #### B10. StepDownSelfRemoved (migrated from P4 event_tx → P2 role_tx) +//! #### B10. StepDownSelfRemoved (migrated from P4 event_tx → P2 internal_event_tx) //! - **B10.1**: Leader — handle_self_removed enqueues BecomeFollower; role transitions to Follower //! -//! #### B11. MembershipApplied (migrated from P4 event_tx → P2 role_tx) +//! #### B11. MembershipApplied (migrated from P4 event_tx → P2 internal_event_tx) //! - **B11.1**: Follower — RoleViolation swallowed; leader-specific logic in membership_change_test //! -//! #### B12. PromoteReadyLearners (migrated from P4 event_tx → P2 role_tx) +//! #### B12. PromoteReadyLearners (migrated from P4 event_tx → P2 internal_event_tx) //! - **B12.1**: Follower — RoleViolation swallowed; leader-specific logic in membership_change_test //! -//! #### B13. LogPurgeCompleted (migrated from P4 event_tx → P2 role_tx) +//! #### B13. LogPurgeCompleted (migrated from P4 event_tx → P2 internal_event_tx) //! - **B13.1**: Follower — dispatch returns Ok, last_purged_index updated //! //! ### C. Leader Initialization Tests (Peer State Setup) @@ -190,40 +190,40 @@ //! //! **Priority Order (P0 > P1 > P2 > P3):** //! - **P0 (Shutdown)**: shutdown_signal.changed() -//! - **F1.1**: Shutdown always processed first, even if tick/role events pending +//! - **F1.1**: Shutdown always processed first, even if tick/internal events pending //! - **F1.2**: Pending events discarded on shutdown //! - **F1.3**: Graceful termination ensures no state corruption //! //! - **P1 (Tick)**: sleep_until(next_deadline) - election timeout / heartbeat //! - **F1.4**: Tick has highest operational priority //! - **F1.5**: Tick drives election timeout and heartbeat cadence -//! - **F1.6**: Tick fires even if role_rx/event_rx have pending messages +//! - **F1.6**: Tick fires even if internal_event_rx/event_rx have pending messages //! -//! - **P2 (RoleEvent)**: role_rx.recv() - internal state transitions -//! - **F1.7**: RoleEvent processed before network events +//! - **P2 (InternalEvent)**: internal_event_rx.recv() - internal state transitions +//! - **F1.7**: InternalEvent processed before network events //! - **F1.8**: Leadership/follower transitions take priority over RPCs -//! - **F1.9**: Multiple role events processed in order (not all at once) +//! - **F1.9**: Multiple internal events processed in order (not all at once) //! -//! - **P3 (RaftEvent)**: event_rx.recv() - network RPCs and responses +//! - **P3 (InboundEvent)**: event_rx.recv() - network RPCs and responses //! - **F1.10**: Network events (AppendEntries, RequestVote) processed last //! - **F1.11**: Responses from followers/candidates deferred until tick completes //! - **F1.12**: This prevents RPC storms from starving timers //! //! #### F2. Concurrent Event Arrival Scenarios //! -//! **Scenario: Tick fires while role_event and raft_event pending** +//! **Scenario: Tick fires while internal_event and inbound_event pending** //! - **F2.1**: Tick processes first (P1 > P2, P3) -//! - **F2.2**: Role event processes next (P2 > P3) -//! - **F2.3**: Raft event processes last +//! - **F2.2**: internal event processes next (P2 > P3) +//! - **F2.3**: inbound event processes last //! - **F2.4**: Starvation prevention: next iteration will service events //! -//! **Scenario: Role event while multiple raft events pending** -//! - **F2.5**: Single role event dequeued and processed -//! - **F2.6**: Only one raft event processed per iteration -//! - **F2.7**: Multiple role events do NOT accumulate in single iteration +//! **Scenario: internal event while multiple inbound events pending** +//! - **F2.5**: Single internal event dequeued and processed +//! - **F2.6**: Only one inbound event processed per iteration +//! - **F2.7**: Multiple internal events do NOT accumulate in single iteration //! //! **Scenario: Shutdown signal during election** -//! - **F2.8**: Shutdown takes priority even if Tick and RoleEvent pending +//! - **F2.8**: Shutdown takes priority even if Tick and InternalEvent pending //! - **F2.9**: Partial election in progress is abandoned //! - **F2.10**: State machine remains consistent (no partial transitions) //! @@ -235,14 +235,14 @@ //! - **F3.1**: Follower election timeout → BecomeCandidate //! - **F3.2**: Candidate timeout → restart election (stay Candidate or become Follower) //! - **F3.3**: Leader timeout → no action (heartbeat continues) -//! - **F3.4**: Timeout NOT missed when role_event/raft_event arrive +//! - **F3.4**: Timeout NOT missed when internal_event/inbound_event arrive //! - **F3.5**: Timeout randomization prevents split brain //! //! #### F4. Complete Election Flow with Event Ordering //! - **F4.1**: Follower → Candidate (election starts via tick) -//! - **F4.2**: Candidate → Leader (majority votes received via raft events) -//! - **F4.3**: Leadership verified (noop entry replicated via raft events) -//! - **F4.4**: RoleEvent BecomeLeader processes before any AppendEntries responses +//! - **F4.2**: Candidate → Leader (majority votes received via inbound events) +//! - **F4.3**: Leadership verified (noop entry replicated via inbound events) +//! - **F4.4**: InternalEvent BecomeLeader processes before any AppendEntries responses //! - **F4.5**: Tick continues at regular intervals during replication //! //! ### G. Error Cases and Edge Cases @@ -260,7 +260,7 @@ //! //! #### G3. Concurrent Event Handling //! - **G3.1**: Multiple listeners receiving notifications simultaneously -//! - **G3.2**: RoleEvents arriving while processing previous event +//! - **G3.2**: InternalEvents arriving while processing previous event //! //! #### G4. Listener Registration Edge Cases //! - **G4.1**: Register listeners after role transitions diff --git a/d-engine-core/src/raft_test/process_raft_events_tests.rs b/d-engine-core/src/raft_test/process_inbound_events_tests.rs similarity index 88% rename from d-engine-core/src/raft_test/process_raft_events_tests.rs rename to d-engine-core/src/raft_test/process_inbound_events_tests.rs index 3ce4e8d8..27717042 100644 --- a/d-engine-core/src/raft_test/process_raft_events_tests.rs +++ b/d-engine-core/src/raft_test/process_inbound_events_tests.rs @@ -1,6 +1,6 @@ -//! # process_raft_events drain loop tests +//! # process_inbound_events drain loop tests //! -//! Tests for the in-loop `matches!()` merge guard in `Raft::process_raft_events`. +//! Tests for the in-loop `matches!()` merge guard in `Raft::process_inbound_events`. //! These tests verify that `merge_append_entries` fires per-iteration when the //! front of the queue is an `AppendEntries` event — not only once at the start //! of the loop (which was the pre-fix behavior flagged in CodeRabbit review #407). @@ -13,12 +13,12 @@ //! Mock expectations are verified on drop at the end of each test. use crate::AppendResponseWithUpdates; -use crate::RoleEvent; +use crate::InternalEvent; use crate::StateUpdate; use crate::maybe_clone_oneshot::MaybeCloneOneshot; use crate::maybe_clone_oneshot::RaftOneshot; use crate::test_utils::MockBuilder; -use crate::{MockElectionCore, MockReplicationCore, RaftEvent, mock_entries}; +use crate::{InboundEvent, MockElectionCore, MockReplicationCore, mock_entries}; use d_engine_proto::server::election::VoteRequest; use d_engine_proto::server::replication::{AppendEntriesRequest, AppendEntriesResponse}; use std::collections::VecDeque; @@ -44,21 +44,21 @@ fn make_request( } } -fn make_ae_event(req: AppendEntriesRequest) -> RaftEvent { +fn make_ae_event(req: AppendEntriesRequest) -> InboundEvent { let (tx, _rx) = MaybeCloneOneshot::new(); - RaftEvent::AppendEntries(req, vec![tx]) + InboundEvent::AppendEntries(req, vec![tx]) } -fn make_fatal_event() -> RoleEvent { - RoleEvent::FatalError { +fn make_fatal_event() -> InternalEvent { + InternalEvent::FatalError { source: "core".to_string(), error: "fatal-error".to_string(), } } -fn make_vr_event() -> RaftEvent { +fn make_vr_event() -> InboundEvent { let (tx, _rx) = MaybeCloneOneshot::new(); - RaftEvent::ReceiveVoteRequest( + InboundEvent::ReceiveVoteRequest( VoteRequest { term: 1, candidate_id: 2, @@ -123,9 +123,9 @@ async fn test_non_ae_prefix_does_not_block_ae_merge_in_drain() { queue.push_back(make_vr_event()); queue.push_back(make_ae_event(make_request(5, 9, 5, 1))); queue.push_back(make_ae_event(make_request(5, 14, 5, 1))); - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; - raft.process_raft_events().await.unwrap(); + raft.process_inbound_events().await.unwrap(); // mock drop verifies: handle_vote_request×1, handle_append_entries×1 } @@ -171,9 +171,9 @@ async fn test_ae_runs_on_both_sides_of_non_ae_are_each_merged() { // second run: entries 20-29 (new entries, continues after first run) queue.push_back(make_ae_event(make_request(5, 19, 5, 1))); queue.push_back(make_ae_event(make_request(5, 24, 5, 1))); - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; - raft.process_raft_events().await.unwrap(); + raft.process_inbound_events().await.unwrap(); } /// Buffer with only non-AE events drains without calling merge and without panicking. @@ -208,9 +208,9 @@ async fn test_all_non_ae_drain_no_merge_no_panic() { let mut queue = VecDeque::new(); queue.push_back(make_vr_event()); queue.push_back(make_vr_event()); - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; - raft.process_raft_events().await.unwrap(); + raft.process_inbound_events().await.unwrap(); } /// A single AE sandwiched between non-AE events is dispatched alone. @@ -250,36 +250,36 @@ async fn test_isolated_ae_between_non_ae_dispatched_alone() { queue.push_back(make_vr_event()); // isolated AE: entries 15-19 queue.push_back(make_ae_event(make_request(5, 14, 5, 1))); - raft.buffered_raft_event = queue; + raft.buffered_inbound_event = queue; - raft.process_raft_events().await.unwrap(); + raft.process_inbound_events().await.unwrap(); } // ----------------------------------------------------------------------- -// process_role_events error propagation +// process_internal_events error propagation // ----------------------------------------------------------------------- -/// A RoleEvent::FatalError in the buffer causes process_role_events to +/// A InternalEvent::FatalError in the buffer causes process_internal_events to /// return Err immediately rather than swallowing the error and continuing. /// /// This guards against the node silently continuing after a fatal signal /// (e.g. state machine worker crash). /// -/// Buffer before: [RoleEvent::FatalError { source: "sm_worker", error: "disk full" }] +/// Buffer before: [InternalEvent::FatalError { source: "sm_worker", error: "disk full" }] /// /// Expected: /// - returns Err(e) where e.is_fatal() == true /// - does NOT return Ok(()) #[tokio::test] -async fn test_process_role_events_propagates_fatal_error() { +async fn test_process_internal_events_propagates_fatal_error() { let (_graceful_tx, graceful_rx) = watch::channel(()); let mut raft = MockBuilder::new(graceful_rx).build_raft(); let mut queue = VecDeque::new(); queue.push_back(make_fatal_event()); - raft.buffered_role_event = queue; + raft.buffered_internal_event = queue; - let err = raft.process_role_events().await.unwrap_err(); + let err = raft.process_internal_events().await.unwrap_err(); assert!(err.is_fatal()); } diff --git a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs index e2babf0e..cac344cd 100644 --- a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs +++ b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs @@ -2,13 +2,13 @@ //! //! **Original location**: raft_test.rs L517-2528 //! **Test count**: 78 tests -//! **Categories**: A (Role Transitions), B (RoleEvent Handling), C (Leader Init), +//! **Categories**: A (Role Transitions), B (InternalEvent Handling), C (Leader Init), //! D (Leadership Verification), F (Event Queue), G (Error Cases), //! H (Bootstrap & Joining) //! //! This is the largest test module covering core Raft consensus behavior: //! - A: Role state machine transitions (25 tests) -//! - B: RoleEvent processing (6 tests within this module) +//! - B: InternalEvent processing (6 tests within this module) //! - C: Leader initialization of peer state (12 tests) //! - D: Leadership verification via noop entry (9 tests) //! - F: Event loop priority and election timeout (12 tests) @@ -28,8 +28,8 @@ use d_engine_proto::common::NodeRole::{Candidate, Follower, Leader, Learner}; use tokio::sync::mpsc; use tokio::sync::watch; +use crate::InternalEvent; use crate::NewCommitData; -use crate::RoleEvent; use crate::test_utils::MockBuilder; use crate::test_utils::MockTypeConfig; @@ -115,23 +115,23 @@ async fn test_role_transition_follower_candidate_leader_follower() { raft.ctx.handlers.replication_handler = replication_core; // Follower cannot directly become Follower - assert!(raft.handle_role_event(RoleEvent::BecomeFollower(None)).await.is_err()); + assert!(raft.handle_internal_event(InternalEvent::BecomeFollower(None)).await.is_err()); assert!(is_follower(raft.role.as_i32())); // Follower → Candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Follower should transition to Candidate"); assert!(is_candidate(raft.role.as_i32())); // Candidate → Leader - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Candidate should transition to Leader"); assert!(is_leader(raft.role.as_i32())); // Leader → Follower - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("Leader should transition to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -147,13 +147,13 @@ async fn test_role_transition_follower_candidate_follower() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Follower should transition to Candidate"); assert!(is_candidate(raft.role.as_i32())); // Candidate → Follower (step down after election failure) - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("Candidate should step down to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -172,13 +172,13 @@ async fn test_role_transition_follower_candidate_leader() { raft.ctx.handlers.replication_handler = replication_core; // Follower → Candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Follower should transition to Candidate"); assert!(is_candidate(raft.role.as_i32())); // Candidate → Leader - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Candidate should transition to Leader"); assert!(is_leader(raft.role.as_i32())); @@ -194,13 +194,13 @@ async fn test_role_transition_candidate_follower() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); // Candidate → Follower (on discovery of higher term) - raft.handle_role_event(RoleEvent::BecomeFollower(Some(2))) + raft.handle_internal_event(InternalEvent::BecomeFollower(Some(2))) .await .expect("Candidate should step down to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -217,13 +217,13 @@ async fn test_role_transition_candidate_candidate_invalid() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); // Candidate → Candidate (invalid - should error) - let result = raft.handle_role_event(RoleEvent::BecomeCandidate).await; + let result = raft.handle_internal_event(InternalEvent::BecomeCandidate).await; assert!( result.is_err(), "Candidate should not transition to Candidate" @@ -244,16 +244,16 @@ async fn test_role_transition_leader_follower() { raft.ctx.handlers.replication_handler = replication_core; // Follower → Candidate → Leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); // Leader → Follower (step down) - raft.handle_role_event(RoleEvent::BecomeFollower(Some(2))) + raft.handle_internal_event(InternalEvent::BecomeFollower(Some(2))) .await .expect("Leader should step down to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -273,16 +273,16 @@ async fn test_role_transition_leader_candidate_invalid() { raft.ctx.handlers.replication_handler = replication_core; // Follower → Candidate → Leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); // Leader → Candidate (invalid - should error) - let result = raft.handle_role_event(RoleEvent::BecomeCandidate).await; + let result = raft.handle_internal_event(InternalEvent::BecomeCandidate).await; assert!(result.is_err(), "Leader should not transition to Candidate"); assert!(is_leader(raft.role.as_i32()), "Should remain Leader"); } @@ -300,16 +300,16 @@ async fn test_role_transition_leader_leader_invalid() { raft.ctx.handlers.replication_handler = replication_core; // Follower → Candidate → Leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); // Leader → Leader (invalid - should error) - let result = raft.handle_role_event(RoleEvent::BecomeLeader).await; + let result = raft.handle_internal_event(InternalEvent::BecomeLeader).await; assert!(result.is_err(), "Leader should not transition to Leader"); assert!(is_leader(raft.role.as_i32()), "Should remain Leader"); } @@ -324,13 +324,13 @@ async fn test_role_transition_learner_follower() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Learner - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); // Learner → Follower (promotion) - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("Learner should transition to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -347,13 +347,13 @@ async fn test_role_transition_learner_candidate_invalid() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Learner - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); // Learner → Candidate (invalid - should error) - let result = raft.handle_role_event(RoleEvent::BecomeCandidate).await; + let result = raft.handle_internal_event(InternalEvent::BecomeCandidate).await; assert!( result.is_err(), "Learner should not transition to Candidate" @@ -371,13 +371,13 @@ async fn test_role_transition_learner_leader_invalid() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Learner - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); // Learner → Leader (invalid - should error) - let result = raft.handle_role_event(RoleEvent::BecomeLeader).await; + let result = raft.handle_internal_event(InternalEvent::BecomeLeader).await; assert!(result.is_err(), "Learner should not transition to Leader"); assert!(is_learner(raft.role.as_i32()), "Should remain Learner"); } @@ -392,13 +392,13 @@ async fn test_role_transition_learner_learner_invalid() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Learner - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); // Learner → Learner (invalid - should error) - let result = raft.handle_role_event(RoleEvent::BecomeLearner).await; + let result = raft.handle_internal_event(InternalEvent::BecomeLearner).await; assert!(result.is_err(), "Learner should not transition to Learner"); assert!(is_learner(raft.role.as_i32()), "Should remain Learner"); } @@ -416,15 +416,15 @@ async fn test_follower_become_with_known_leader() { // Follower with known leader let leader_id = 3; - raft.handle_role_event(RoleEvent::BecomeFollower(Some(leader_id))) + raft.handle_internal_event(InternalEvent::BecomeFollower(Some(leader_id))) .await .expect_err("Initial follower should not be able to become follower again"); // First transition to candidate, then back to follower with known leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeFollower(Some(leader_id))) + raft.handle_internal_event(InternalEvent::BecomeFollower(Some(leader_id))) .await .expect("Should transition to Follower with known leader"); assert!(is_follower(raft.role.as_i32())); @@ -440,12 +440,12 @@ async fn test_follower_become_without_leader() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // First become candidate, then follower without known leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("Should transition to Follower without known leader"); assert!(is_follower(raft.role.as_i32())); @@ -462,13 +462,13 @@ async fn test_follower_state_reset_voted_for() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower → Candidate (sets voted_for to self) - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); // Candidate → Follower (should clear voted_for) - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("Should transition to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -492,7 +492,7 @@ async fn test_candidate_become_increments_term() { let initial_term = raft.role.current_term(); // Transition to candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); @@ -518,7 +518,7 @@ async fn test_candidate_become_votes_for_self() { let _node_id = raft.ctx.node_id; // Transition to candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); @@ -536,7 +536,7 @@ async fn test_candidate_become_clears_leader() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // First make candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); @@ -559,10 +559,10 @@ async fn test_leader_become_marks_vote_committed() { raft.ctx.handlers.replication_handler = replication_core; // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -584,10 +584,10 @@ async fn test_leader_become_initializes_peer_indices() { raft.ctx.handlers.replication_handler = replication_core; // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -608,10 +608,10 @@ async fn test_leader_become_initializes_metadata_cache() { raft.ctx.handlers.replication_handler = replication_core; // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -632,10 +632,10 @@ async fn test_leader_become_sends_noop_entry() { raft.ctx.handlers.replication_handler = replication_core; // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -655,10 +655,10 @@ async fn test_leader_verification_succeeds() { raft.ctx.handlers.replication_handler = replication_core; // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -695,10 +695,10 @@ async fn test_leader_verification_fails_downgrades() { raft.ctx.handlers.replication_handler = replication_handler; // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -727,10 +727,10 @@ async fn test_leader_single_node_cluster() { raft.ctx.membership = Arc::new(membership); // Transition to leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -750,7 +750,7 @@ async fn test_learner_become_non_voting() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Transition to learner - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); @@ -767,13 +767,13 @@ async fn test_learner_promotion_to_follower() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Transition to learner - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); // Promote learner to follower - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("Learner should be promoted to Follower"); assert!(is_follower(raft.role.as_i32())); @@ -789,7 +789,7 @@ async fn test_multiple_learners_tracking() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Transition to learner (this learner instance is tracked) - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); @@ -797,7 +797,7 @@ async fn test_multiple_learners_tracking() { } // ============================================================================ -// B. ROLEEVENT HANDLING TESTS (Individual Event Processing) +// B. INTERNAL EVENT HANDLING TESTS (Individual Event Processing) // ============================================================================ // B1. BecomeFollower Event Tests @@ -816,13 +816,13 @@ async fn test_become_follower_sends_leader_notification() { raft.register_leader_change_listener(leader_tx); // First become candidate - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); // Then become follower with a known leader let leader_id = 2; - raft.handle_role_event(RoleEvent::BecomeFollower(Some(leader_id))) + raft.handle_internal_event(InternalEvent::BecomeFollower(Some(leader_id))) .await .expect("Should become Follower"); assert!(is_follower(raft.role.as_i32())); @@ -848,7 +848,7 @@ async fn test_become_candidate_clears_leader_notification() { raft.register_leader_change_listener(leader_tx); // Become candidate (no leader in this state) - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); assert!(is_candidate(raft.role.as_i32())); @@ -875,10 +875,10 @@ async fn test_become_leader_full_initialization() { raft.register_leader_change_listener(leader_tx); // Become candidate then leader - raft.handle_role_event(RoleEvent::BecomeCandidate) + raft.handle_internal_event(InternalEvent::BecomeCandidate) .await .expect("Should become Candidate"); - raft.handle_role_event(RoleEvent::BecomeLeader) + raft.handle_internal_event(InternalEvent::BecomeLeader) .await .expect("Should become Leader"); assert!(is_leader(raft.role.as_i32())); @@ -901,16 +901,16 @@ async fn test_leader_ready_notification_only_after_noop_committed() { let (leader_tx, mut leader_rx) = watch::channel(None); raft.register_leader_change_listener(leader_tx); - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); - // In the async design, BecomeLeader queues RoleEvent::NoopCommitted via role_tx. + // In the async design, BecomeLeader queues InternalEvent::NoopCommitted via internal_event_tx. // The Raft loop would process it, but in unit tests we drive events manually. // Single-voter path: initiate_noop_commit fires drain_commit_actions synchronously - // (via Phase 4 → handle_log_flushed), sending NoopCommitted to role_tx. + // (via Phase 4 → handle_log_flushed), sending NoopCommitted to internal_event_tx. // Process it now to complete the leader ready notification. let term = raft.role.current_term(); - raft.handle_role_event(RoleEvent::NoopCommitted { term }).await.unwrap(); + raft.handle_internal_event(InternalEvent::NoopCommitted { term }).await.unwrap(); // Must receive Some(leader_id): noop committed successfully, cluster is ready leader_rx.changed().await.expect("Should receive leader ready notification"); @@ -946,8 +946,8 @@ async fn test_leader_ready_notification_suppressed_when_noop_fails() { let (leader_tx, mut leader_rx) = watch::channel(None); raft.register_leader_change_listener(leader_tx); - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); // Must NOT receive Some(leader_id). Channel starts as None; BecomeFollower writes None // again — send_if_modified skips (no change), so changed() never fires. Timeout = correct. @@ -978,7 +978,7 @@ async fn test_become_learner_no_leader() { raft.register_leader_change_listener(leader_tx); // Become learner (no leader in this state) - raft.handle_role_event(RoleEvent::BecomeLearner) + raft.handle_internal_event(InternalEvent::BecomeLearner) .await .expect("Should become Learner"); assert!(is_learner(raft.role.as_i32())); @@ -1005,7 +1005,7 @@ async fn test_notify_new_commit_index_broadcasts() { // Send NotifyNewCommitIndex event let new_commit_index = 10; - raft.handle_role_event(RoleEvent::NotifyNewCommitIndex(NewCommitData { + raft.handle_internal_event(InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index, role: Follower.into(), current_term: 1, @@ -1036,7 +1036,7 @@ async fn test_leader_discovered_no_state_change() { let initial_role = raft.role.as_i32(); // Send LeaderDiscovered event - raft.handle_role_event(RoleEvent::LeaderDiscovered(3, 5)) + raft.handle_internal_event(InternalEvent::LeaderDiscovered(3, 5)) .await .expect("Should handle LeaderDiscovered"); @@ -1063,9 +1063,9 @@ async fn test_reprocess_event_requeues() { // B8–B13. New Lifecycle Event Dispatch Tests // -// Six RoleEvent variants were migrated from bounded event_tx (P4) to unbounded -// role_tx (P2) to eliminate deadlock when the P4 channel is saturated by inbound RPCs. -// These tests verify that each dispatch arm in handle_role_event routes correctly +// Six InternalEvent variants were migrated from bounded event_tx (P4) to unbounded +// internal_event_tx (P2) to eliminate deadlock when the P4 channel is saturated by inbound RPCs. +// These tests verify that each dispatch arm in handle_internal_event routes correctly // — no todo!() panic, correct return value, and the observable role-level outcome. // // Handler-level behaviour (flag mutations, purge logic, metadata refresh) is already @@ -1081,7 +1081,7 @@ async fn test_create_snapshot_event_dispatch_on_follower() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); assert!(is_follower(raft.role.as_i32())); - raft.handle_role_event(RoleEvent::CreateSnapshotEvent) + raft.handle_internal_event(InternalEvent::CreateSnapshotEvent) .await .expect("CreateSnapshotEvent on Follower must not return a fatal error"); @@ -1098,11 +1098,11 @@ async fn test_create_snapshot_event_dispatch_on_candidate() { let (_graceful_tx, graceful_rx) = watch::channel(()); let mut raft = MockBuilder::new(graceful_rx).build_raft(); - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); assert!(is_candidate(raft.role.as_i32())); // RoleViolation is non-fatal — dispatch swallows it and returns Ok. - raft.handle_role_event(RoleEvent::CreateSnapshotEvent) + raft.handle_internal_event(InternalEvent::CreateSnapshotEvent) .await .expect("CreateSnapshotEvent on Candidate must return Ok (RoleViolation swallowed)"); @@ -1125,7 +1125,7 @@ async fn test_snapshot_created_err_dispatch_on_follower() { // Use a non-fatal snapshot error so the dispatch does not propagate it. let err_result = Err(crate::SnapshotError::OperationFailed("simulated failure".into()).into()); - raft.handle_role_event(RoleEvent::SnapshotCreated(err_result)) + raft.handle_internal_event(InternalEvent::SnapshotCreated(err_result)) .await .expect("SnapshotCreated(Err) on Follower must return Ok (non-fatal error swallowed)"); @@ -1135,7 +1135,7 @@ async fn test_snapshot_created_err_dispatch_on_follower() { /// B10 — StepDownSelfRemoved on Leader causes role transition to Follower /// /// This is the most behaviourally significant of the six new events at the integration -/// level. handle_self_removed posts BecomeFollower(None) onto role_tx so the main loop +/// level. handle_self_removed posts BecomeFollower(None) onto internal_event_tx so the main loop /// can process the transition on the next iteration. In unit tests we drive that queued /// event manually to verify the complete Leader → StepDownSelfRemoved → Follower path. #[tokio::test] @@ -1147,20 +1147,20 @@ async fn test_step_down_self_removed_leader_transitions_to_follower() { raft.ctx.handlers.replication_handler = replication_core; // Establish Leader role. - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Step 1: dispatch StepDownSelfRemoved — handle_self_removed enqueues - // BecomeFollower(None) on the internal role_tx and returns Ok. - raft.handle_role_event(RoleEvent::StepDownSelfRemoved) + // BecomeFollower(None) on the internal internal_event_tx and returns Ok. + raft.handle_internal_event(InternalEvent::StepDownSelfRemoved) .await .expect("StepDownSelfRemoved on Leader must not return a fatal error"); // Step 2: simulate the main loop consuming the queued BecomeFollower(None). // In production this is driven by the biased select! loop; in unit tests // we drive it manually to verify the full transition completes. - raft.handle_role_event(RoleEvent::BecomeFollower(None)) + raft.handle_internal_event(InternalEvent::BecomeFollower(None)) .await .expect("BecomeFollower queued by StepDownSelfRemoved must succeed"); @@ -1180,7 +1180,7 @@ async fn test_membership_applied_dispatch_on_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); let mut raft = MockBuilder::new(graceful_rx).build_raft(); - raft.handle_role_event(RoleEvent::MembershipApplied) + raft.handle_internal_event(InternalEvent::MembershipApplied) .await .expect("MembershipApplied on Follower must return Ok (RoleViolation swallowed)"); @@ -1197,7 +1197,7 @@ async fn test_promote_ready_learners_dispatch_on_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); let mut raft = MockBuilder::new(graceful_rx).build_raft(); - raft.handle_role_event(RoleEvent::PromoteReadyLearners) + raft.handle_internal_event(InternalEvent::PromoteReadyLearners) .await .expect("PromoteReadyLearners on Follower must return Ok (RoleViolation swallowed)"); @@ -1215,9 +1215,12 @@ async fn test_log_purge_completed_dispatch_on_follower() { let (_graceful_tx, graceful_rx) = watch::channel(()); let mut raft = MockBuilder::new(graceful_rx).build_raft(); - raft.handle_role_event(RoleEvent::LogPurgeCompleted(LogId { term: 1, index: 50 })) - .await - .expect("LogPurgeCompleted on Follower must not return a fatal error"); + raft.handle_internal_event(InternalEvent::LogPurgeCompleted(LogId { + term: 1, + index: 50, + })) + .await + .expect("LogPurgeCompleted on Follower must not return a fatal error"); assert!(is_follower(raft.role.as_i32())); } @@ -1242,8 +1245,8 @@ async fn test_leader_init_single_peer() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Peer indices are initialized correctly } @@ -1261,8 +1264,8 @@ async fn test_leader_init_two_peers() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Both peers are initialized with correct indices } @@ -1280,8 +1283,8 @@ async fn test_leader_init_three_peers() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // All three peers are initialized } @@ -1299,8 +1302,8 @@ async fn test_leader_init_five_peers() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // All five peers are initialized } @@ -1318,8 +1321,8 @@ async fn test_leader_init_match_index_zero() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Match index is initialized to 0 for all peers } @@ -1337,8 +1340,8 @@ async fn test_leader_init_includes_learners() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Learner peers are initialized separately } @@ -1356,8 +1359,8 @@ async fn test_leader_init_mixed_voters_learners() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Both voters and learners are initialized together } @@ -1377,8 +1380,8 @@ async fn test_leader_metadata_cache_replication_peers() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Metadata cache includes all replication peers } @@ -1396,8 +1399,8 @@ async fn test_leader_metadata_cache_voters() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Metadata cache includes voters for quorum calculation } @@ -1415,8 +1418,8 @@ async fn test_leader_metadata_cache_all_active() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Cache includes all active peers } @@ -1434,8 +1437,8 @@ async fn test_leader_metadata_cache_learner_separation() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Learners are tracked separately from voters } @@ -1461,8 +1464,8 @@ async fn test_leader_metadata_cache_single_node() { raft.ctx.membership = Arc::new(membership); // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Single-node cluster metadata cache has no peers } @@ -1487,8 +1490,8 @@ async fn test_leadership_verification_success() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Leadership verification succeeds with majority confirmation } @@ -1525,8 +1528,8 @@ async fn test_leadership_verification_failure_downgrades() { raft.ctx.handlers.replication_handler = replication_handler; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // On verification failure, leader should downgrade } @@ -1552,8 +1555,8 @@ async fn test_leadership_verification_single_node() { raft.ctx.membership = Arc::new(membership); // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Single-node verification always succeeds } @@ -1573,8 +1576,8 @@ async fn test_quorum_exactly_majority_3node() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // 2 out of 3 is exactly the majority } @@ -1592,8 +1595,8 @@ async fn test_quorum_5node_minority_failure() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // 5-node cluster needs 3+ responses for quorum } @@ -1628,8 +1631,8 @@ async fn test_network_partition_minority_loses_leadership() { raft.ctx.handlers.replication_handler = replication_handler; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Minority partition should lose leadership } @@ -1658,8 +1661,8 @@ async fn test_leader_change_notification_correctness() { raft.register_leader_change_listener(leader_tx); // Become candidate then leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); // Leader should send notification with self as leader assert!(is_leader(raft.role.as_i32())); @@ -1683,7 +1686,7 @@ async fn test_new_commit_notification_correctness() { // Send commit notification let new_commit_index = 25; - raft.handle_role_event(RoleEvent::NotifyNewCommitIndex(NewCommitData { + raft.handle_internal_event(InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index, role: Follower.into(), current_term: 5, @@ -1715,12 +1718,12 @@ async fn test_role_transition_notification() { raft.register_role_transition_listener(tx); // Transition: Follower → Candidate - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); let role = rx.recv().await.expect("Should receive role transition"); assert!(is_candidate(role)); // Transition: Candidate → Leader - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); let role = rx.recv().await.expect("Should receive role transition"); assert!(is_leader(role)); } @@ -1734,7 +1737,7 @@ async fn test_role_transition_notification() { /// Test: Shutdown has highest priority (P0) /// /// Verifies shutdown signal is processed before all other events. -/// Even with pending tick/role/raft events, shutdown is first. +/// Even with pending tick/role/inbound events, shutdown is first. /// See F1.1-F1.3 in test scenarios. #[tokio::test] async fn test_event_priority_p0_shutdown() { @@ -1749,7 +1752,7 @@ async fn test_event_priority_p0_shutdown() { /// Test: Tick has highest operational priority (P1) /// /// Verifies tick (election timeout/heartbeat) is prioritized. -/// Tick fires before role events and raft events. +/// Tick fires before internal events and inbound events. /// See F1.4-F1.6 in test scenarios. #[tokio::test] async fn test_event_priority_p1_tick() { @@ -1757,10 +1760,10 @@ async fn test_event_priority_p1_tick() { let _raft = MockBuilder::new(graceful_rx).build_raft(); // Tick (election timeout/heartbeat) has highest operational priority - // Even with role and raft events pending, tick is processed first + // Even with role and inbound events pending, tick is processed first } -/// Test: RoleEvent processed before RaftEvent (P2 > P3) +/// Test: InternalEvent processed before InboundEvent (P2 > P3) /// /// Verifies role state transitions take priority over network events. /// Leadership changes processed before AppendEntries responses. @@ -1770,11 +1773,11 @@ async fn test_event_priority_p2_role_before_p3_raft() { let (_graceful_tx, graceful_rx) = watch::channel(()); let _raft = MockBuilder::new(graceful_rx).build_raft(); - // RoleEvent has priority over RaftEvent + // InternalEvent has priority over InboundEvent // State transitions processed before network events } -/// Test: RaftEvent has lowest priority (P3) +/// Test: InboundEvent has lowest priority (P3) /// /// Verifies network events are processed last. /// This prevents RPC storms from starving timers. @@ -1784,15 +1787,15 @@ async fn test_event_priority_p3_raft_last() { let (_graceful_tx, graceful_rx) = watch::channel(()); let _raft = MockBuilder::new(graceful_rx).build_raft(); - // RaftEvent (network events) has lowest priority + // InboundEvent (network events) has lowest priority // Prevents RPC storms from starving timers } // F2. Concurrent Event Arrival Scenarios -/// Test: Tick fires with role_event and raft_event pending +/// Test: Tick fires with internal_event and inbound_event pending /// -/// Verifies tick processes first, then role event, then raft event. +/// Verifies tick processes first, then internal event, then inbound event. /// See F2.1-F2.4 in test scenarios. #[tokio::test] async fn test_concurrent_events_tick_role_raft() { @@ -1801,13 +1804,13 @@ async fn test_concurrent_events_tick_role_raft() { // Concurrent events are processed in priority order: // 1. Tick (P1) - // 2. RoleEvent (P2) - // 3. RaftEvent (P3) + // 2. InternalEvent (P2) + // 3. InboundEvent (P3) } -/// Test: Role event with multiple raft events pending +/// Test: internal event with multiple inbound events pending /// -/// Verifies one role event and one raft event per iteration. +/// Verifies one internal event and one inbound event per iteration. /// Prevents starvation while ensuring progress. /// See F2.5-F2.7 in test scenarios. #[tokio::test] @@ -1815,7 +1818,7 @@ async fn test_concurrent_events_role_multiple_raft() { let (_graceful_tx, graceful_rx) = watch::channel(()); let _raft = MockBuilder::new(graceful_rx).build_raft(); - // Process one role event and one raft event per iteration + // Process one internal event and one inbound event per iteration // Prevents starvation while maintaining fair progress } @@ -1860,7 +1863,7 @@ async fn test_election_timeout_candidate_restart() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Become candidate - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); assert!(is_candidate(raft.role.as_i32())); // Candidate timeout restarts election } @@ -1878,8 +1881,8 @@ async fn test_election_timeout_leader_no_action() { raft.ctx.handlers.replication_handler = replication_core; // Become leader - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Leader timeout doesn't trigger election, continues heartbeat } @@ -1915,11 +1918,11 @@ async fn test_complete_election_flow() { assert!(is_follower(raft.role.as_i32())); // Follower → Candidate (tick timeout) - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); assert!(is_candidate(raft.role.as_i32())); // Candidate → Leader (majority votes received) - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); // Complete election flow with proper event ordering @@ -1939,15 +1942,15 @@ async fn test_invalid_transitions_return_error() { let mut raft = MockBuilder::new(graceful_rx).build_raft(); // Follower cannot transition to Follower - assert!(raft.handle_role_event(RoleEvent::BecomeFollower(None)).await.is_err()); + assert!(raft.handle_internal_event(InternalEvent::BecomeFollower(None)).await.is_err()); // Candidate → Candidate (invalid) - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); - assert!(raft.handle_role_event(RoleEvent::BecomeCandidate).await.is_err()); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + assert!(raft.handle_internal_event(InternalEvent::BecomeCandidate).await.is_err()); // Learner → Learner (invalid) - raft.handle_role_event(RoleEvent::BecomeLearner).await.unwrap(); - assert!(raft.handle_role_event(RoleEvent::BecomeLearner).await.is_err()); + raft.handle_internal_event(InternalEvent::BecomeLearner).await.unwrap(); + assert!(raft.handle_internal_event(InternalEvent::BecomeLearner).await.is_err()); } /// Test: State unchanged after failed transition @@ -1963,7 +1966,7 @@ async fn test_failed_transition_state_unchanged() { let initial_role = raft.role.as_i32(); // Try invalid transition - let _ = raft.handle_role_event(RoleEvent::BecomeFollower(None)).await; + let _ = raft.handle_internal_event(InternalEvent::BecomeFollower(None)).await; // State should be unchanged assert_eq!(raft.role.as_i32(), initial_role); @@ -1991,11 +1994,11 @@ async fn test_single_node_cluster_election() { raft.ctx.membership = Arc::new(membership); // Become candidate - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); assert!(is_candidate(raft.role.as_i32())); // Single node becomes leader immediately - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); assert!(is_leader(raft.role.as_i32())); } @@ -2017,7 +2020,7 @@ async fn test_multiple_listeners_concurrent() { raft.register_new_commit_listener(tx3); // Send notification - raft.handle_role_event(RoleEvent::NotifyNewCommitIndex(NewCommitData { + raft.handle_internal_event(InternalEvent::NotifyNewCommitIndex(NewCommitData { new_commit_index: 5, role: Follower.into(), current_term: 1, @@ -2044,14 +2047,14 @@ async fn test_late_listener_registration() { raft.ctx.handlers.replication_handler = replication_core; // First transition - raft.handle_role_event(RoleEvent::BecomeCandidate).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); // Now register listener let (tx, mut rx) = mpsc::unbounded_channel::(); raft.register_role_transition_listener(tx); // Next transition should be received - raft.handle_role_event(RoleEvent::BecomeLeader).await.unwrap(); + raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); let role = rx.recv().await.expect("Should receive late registration"); assert!(is_leader(role)); } diff --git a/d-engine-core/src/replication/mod.rs b/d-engine-core/src/replication/mod.rs index cded9e9a..7cb213b2 100644 --- a/d-engine-core/src/replication/mod.rs +++ b/d-engine-core/src/replication/mod.rs @@ -166,7 +166,7 @@ where /// 3. CALLER MUST: /// - Check `response.term_update` for term conflicts /// - If higher term exists, transition to Follower - /// - Apply other state updates via role_tx + /// - Apply other state updates via internal_event_tx async fn handle_append_entries( &self, request: AppendEntriesRequest, diff --git a/d-engine-core/src/state_machine_handler/worker.rs b/d-engine-core/src/state_machine_handler/worker.rs index 51eca20a..e84ad4ca 100644 --- a/d-engine-core/src/state_machine_handler/worker.rs +++ b/d-engine-core/src/state_machine_handler/worker.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use tokio::sync::{mpsc, watch}; use tracing::{debug, error, info}; +use crate::InternalEvent; use crate::Result; -use crate::RoleEvent; use crate::StateMachineHandler; use crate::TypeConfig; use crate::alias::SMHOF; @@ -20,7 +20,7 @@ use crate::alias::SMHOF; pub struct StateMachineWorker { state_machine_handler: Arc>, sm_apply_rx: mpsc::UnboundedReceiver>, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, shutdown_signal: watch::Receiver<()>, node_id: u32, } @@ -30,13 +30,13 @@ impl StateMachineWorker { node_id: u32, state_machine_handler: Arc>, sm_apply_rx: mpsc::UnboundedReceiver>, - role_tx: mpsc::UnboundedSender, + internal_event_tx: mpsc::UnboundedSender, shutdown_signal: watch::Receiver<()>, ) -> Self { Self { state_machine_handler, sm_apply_rx, - role_tx, + internal_event_tx, shutdown_signal, node_id, } @@ -50,7 +50,7 @@ impl StateMachineWorker { debug!("[Node-{}] SM Worker started", self.node_id); let node_id = self.node_id; let state_machine_handler = self.state_machine_handler; - let role_tx = self.role_tx; + let internal_event_tx = self.internal_event_tx; let mut sm_apply_rx = self.sm_apply_rx; let mut shutdown_signal = self.shutdown_signal.clone(); let mut shutdown = false; @@ -67,7 +67,7 @@ impl StateMachineWorker { entries = sm_apply_rx.recv() => { match entries { Some(entries) => { - Self::apply_and_notify(&node_id, &state_machine_handler, &role_tx, entries).await?; + Self::apply_and_notify(&node_id, &state_machine_handler, &internal_event_tx, entries).await?; } None => { debug!("[Node-{}] SM Worker: apply channel closed", node_id); @@ -86,7 +86,13 @@ impl StateMachineWorker { // Graceful drain: process remaining entries after shutdown signal debug!("[Node-{}] SM Worker draining pending applies", node_id); while let Ok(entries) = sm_apply_rx.try_recv() { - Self::apply_and_notify(&node_id, &state_machine_handler, &role_tx, entries).await?; + Self::apply_and_notify( + &node_id, + &state_machine_handler, + &internal_event_tx, + entries, + ) + .await?; } info!("[Node-{}] SM Worker shutdown complete", node_id); @@ -96,7 +102,7 @@ impl StateMachineWorker { async fn apply_and_notify( node_id: &u32, state_machine_handler: &Arc>, - role_tx: &mpsc::UnboundedSender, + internal_event_tx: &mpsc::UnboundedSender, entries: Vec, ) -> Result<()> { // Apply all entries at once (no chunking) @@ -108,10 +114,10 @@ impl StateMachineWorker { node_id, last.index ); - // Send via role_tx (P2, unbounded) to prevent priority inversion: + // Send via internal_event_tx (P2, unbounded) to prevent priority inversion: // ApplyCompleted is internal (driven by commit) and must not be // starved by external RPCs on event_tx (P4, bounded). - if let Err(e) = role_tx.send(RoleEvent::ApplyCompleted { + if let Err(e) = internal_event_tx.send(InternalEvent::ApplyCompleted { last_index: last.index, results, }) { @@ -129,9 +135,9 @@ impl StateMachineWorker { Err(e) => { error!("[Node-{}] SM apply failed: {:?}", node_id, e); - // Send FatalError via role_tx (unbounded) — must not block waiting for + // Send FatalError via internal_event_tx (unbounded) — must not block waiting for // space on the bounded event_tx when the node is already in fatal state. - if let Err(send_err) = role_tx.send(RoleEvent::FatalError { + if let Err(send_err) = internal_event_tx.send(InternalEvent::FatalError { source: "StateMachine".to_string(), error: format!("{e:?}"), }) { diff --git a/d-engine-core/src/state_machine_handler/worker_test.rs b/d-engine-core/src/state_machine_handler/worker_test.rs index bf3aa74e..5f6b7331 100644 --- a/d-engine-core/src/state_machine_handler/worker_test.rs +++ b/d-engine-core/src/state_machine_handler/worker_test.rs @@ -2,8 +2,8 @@ //! //! Tests verify SM Worker's core responsibilities: //! - Apply entries asynchronously via apply_chunk -//! - Send ApplyCompleted events on success via role_tx (P2) -//! - Send FatalError events on apply failures via role_tx (P2) +//! - Send ApplyCompleted events on success via internal_event_tx (P2) +//! - Send FatalError events on apply failures via internal_event_tx (P2) //! - Gracefully drain remaining entries on shutdown use std::sync::Arc; @@ -12,7 +12,7 @@ use d_engine_proto::common::Entry; use tokio::sync::{mpsc, watch}; use super::StateMachineWorker; -use crate::{Error, MockStateMachineHandler, MockTypeConfig, RoleEvent}; +use crate::{Error, InternalEvent, MockStateMachineHandler, MockTypeConfig}; /// Helper: Create test entry fn create_test_entry( @@ -32,7 +32,7 @@ fn create_test_entry( /// /// # Test Objective /// Verify that when apply_chunk succeeds, SM Worker sends -/// RoleEvent::ApplyCompleted with correct results via role_tx (P2, unbounded). +/// InternalEvent::ApplyCompleted with correct results via internal_event_tx (P2, unbounded). /// /// # Given /// - Mock StateMachineHandler with successful apply_chunk @@ -42,7 +42,7 @@ fn create_test_entry( /// - apply_chunk returns Ok(results) /// /// # Then -/// - RoleEvent::ApplyCompleted is sent to role_tx +/// - InternalEvent::ApplyCompleted is sent to internal_event_tx /// - Event contains correct last_index and results #[tokio::test] async fn test_apply_success_sends_apply_completed() { @@ -54,14 +54,14 @@ async fn test_apply_success_sends_apply_completed() { }); let (sm_apply_tx, sm_apply_rx) = mpsc::unbounded_channel(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); let (_shutdown_tx, shutdown_rx) = watch::channel(()); let worker = StateMachineWorker::::new( 1, Arc::new(mock_smh), sm_apply_rx, - role_tx, + internal_event_tx, shutdown_rx, ); @@ -73,8 +73,13 @@ async fn test_apply_success_sends_apply_completed() { .send(vec![create_test_entry(1, 1), create_test_entry(2, 1)]) .unwrap(); - match tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()).await { - Ok(Some(RoleEvent::ApplyCompleted { + match tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await + { + Ok(Some(InternalEvent::ApplyCompleted { last_index, results, })) => { @@ -93,7 +98,7 @@ async fn test_apply_success_sends_apply_completed() { /// /// # Test Objective /// Verify that when apply_chunk fails, SM Worker sends -/// RoleEvent::FatalError via role_tx (P2, unbounded) and exits with error. +/// InternalEvent::FatalError via internal_event_tx (P2, unbounded) and exits with error. /// /// # Given /// - Mock StateMachineHandler with failing apply_chunk @@ -102,7 +107,7 @@ async fn test_apply_success_sends_apply_completed() { /// - apply_chunk returns Err(Error::Fatal) /// /// # Then -/// - RoleEvent::FatalError is sent to role_tx +/// - InternalEvent::FatalError is sent to internal_event_tx /// - Worker returns error (exits run loop) #[tokio::test] async fn test_apply_failure_sends_fatal_error() { @@ -114,14 +119,14 @@ async fn test_apply_failure_sends_fatal_error() { }); let (sm_apply_tx, sm_apply_rx) = mpsc::unbounded_channel(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); let (_shutdown_tx, shutdown_rx) = watch::channel(()); let worker = StateMachineWorker::::new( 1, Arc::new(mock_smh), sm_apply_rx, - role_tx, + internal_event_tx, shutdown_rx, ); @@ -129,8 +134,13 @@ async fn test_apply_failure_sends_fatal_error() { sm_apply_tx.send(vec![create_test_entry(1, 1)]).unwrap(); - match tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()).await { - Ok(Some(RoleEvent::FatalError { source, error })) => { + match tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await + { + Ok(Some(InternalEvent::FatalError { source, error })) => { assert_eq!(source, "StateMachine"); assert!( error.contains("Disk failure") || error.contains("storage"), @@ -156,7 +166,7 @@ async fn test_apply_failure_sends_fatal_error() { /// pending entries before exit (no data loss). /// /// # Success Criteria -/// - All 3 entries applied (verified via ApplyCompleted count on role_rx) +/// - All 3 entries applied (verified via ApplyCompleted count on internal_event_rx) /// - Worker run() returns Ok(()) #[tokio::test] async fn test_shutdown_drains_remaining_entries() { @@ -168,14 +178,14 @@ async fn test_shutdown_drains_remaining_entries() { }); let (sm_apply_tx, sm_apply_rx) = mpsc::unbounded_channel(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); let (shutdown_tx, shutdown_rx) = watch::channel(()); let worker = StateMachineWorker::::new( 1, Arc::new(mock_smh), sm_apply_rx, - role_tx, + internal_event_tx, shutdown_rx, ); @@ -190,8 +200,13 @@ async fn test_shutdown_drains_remaining_entries() { let mut apply_count = 0; while apply_count < 3 { - match tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()).await { - Ok(Some(RoleEvent::ApplyCompleted { .. })) => apply_count += 1, + match tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await + { + Ok(Some(InternalEvent::ApplyCompleted { .. })) => apply_count += 1, Ok(Some(other)) => panic!("Expected ApplyCompleted, got {other:?}"), Ok(None) => break, Err(_) => panic!("Timeout waiting for ApplyCompleted events"), @@ -215,14 +230,14 @@ async fn test_channel_closed_worker_exits() { let mock_smh = MockStateMachineHandler::new(); let (sm_apply_tx, sm_apply_rx) = mpsc::unbounded_channel(); - let (role_tx, _role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); let (_shutdown_tx, shutdown_rx) = watch::channel(()); let worker = StateMachineWorker::::new( 1, Arc::new(mock_smh), sm_apply_rx, - role_tx, + internal_event_tx, shutdown_rx, ); @@ -243,7 +258,7 @@ async fn test_channel_closed_worker_exits() { /// Verify SM Worker processes batches in order, last_index increases monotonically. /// /// # Success Criteria -/// - 3 ApplyCompleted events received on role_rx with last_index: 2, 4, 6 +/// - 3 ApplyCompleted events received on internal_event_rx with last_index: 2, 4, 6 #[tokio::test] async fn test_multiple_batches_sequential_processing() { let mut mock_smh = MockStateMachineHandler::new(); @@ -254,14 +269,14 @@ async fn test_multiple_batches_sequential_processing() { }); let (sm_apply_tx, sm_apply_rx) = mpsc::unbounded_channel(); - let (role_tx, mut role_rx) = mpsc::unbounded_channel::(); + let (internal_event_tx, mut internal_event_rx) = mpsc::unbounded_channel::(); let (_shutdown_tx, shutdown_rx) = watch::channel(()); let worker = StateMachineWorker::::new( 1, Arc::new(mock_smh), sm_apply_rx, - role_tx, + internal_event_tx, shutdown_rx, ); @@ -280,8 +295,13 @@ async fn test_multiple_batches_sequential_processing() { .unwrap(); for expected_index in [2u64, 4, 6] { - match tokio::time::timeout(std::time::Duration::from_millis(100), role_rx.recv()).await { - Ok(Some(RoleEvent::ApplyCompleted { last_index, .. })) => { + match tokio::time::timeout( + std::time::Duration::from_millis(100), + internal_event_rx.recv(), + ) + .await + { + Ok(Some(InternalEvent::ApplyCompleted { last_index, .. })) => { assert_eq!( last_index, expected_index, "last_index should match expected" diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index ec986ceb..9a2eb590 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -184,7 +184,7 @@ impl TermSegments { /// IO tasks for the dedicated raft-io thread. /// /// All blocking storage operations route through this channel so they never run -/// on tokio worker threads or the Raft event loop. +/// on tokio worker threads or the inbound event loop. #[derive(Debug)] pub enum IOTask { /// Atomically truncate from `truncate_from` then persist `new_entries`. @@ -202,7 +202,7 @@ pub enum IOTask { done: oneshot::Sender<()>, }, /// Reset log storage (snapshot install). Routes through IO thread so reset - /// never runs on the Raft event loop. + /// never runs on the inbound event loop. Reset { done: oneshot::Sender> }, /// Persist any pending entries then fsync. The IO thread sends the fsync /// result (Ok or Err) back to the caller via the oneshot channel. @@ -269,9 +269,9 @@ where pub(crate) command_sender: mpsc::UnboundedSender, // --- P0: LogFlushed event notification --- - // Sends RoleEvent::LogFlushed(durable) to Raft loop after each fsync. - // None in tests; Some(role_tx) in production. - log_flush_tx: Option>, + // Sends InternalEvent::LogFlushed(durable) to Raft loop after each fsync. + // None in tests; Some(internal_event_tx) in production. + log_flush_tx: Option>, // IO thread handle — set by start(), used by close() to join before returning. io_thread_handle: std::sync::Mutex>>, @@ -635,7 +635,7 @@ where self.last_purged_term.store(cutoff_index.term, Ordering::Release); self.last_purged_index.store(cutoff_index.index, Ordering::Release); - // Route purge through the IO thread so it never blocks the Raft event loop. + // Route purge through the IO thread so it never blocks the inbound event loop. // Also writes the purge boundary to META_CF in the RocksDB implementation. let (done_tx, done_rx) = oneshot::channel(); self.command_sender @@ -822,7 +822,7 @@ where pub fn start( mut self, receiver: mpsc::UnboundedReceiver, - log_flush_tx: Option>, + log_flush_tx: Option>, ) -> Arc { self.log_flush_tx = log_flush_tx; let arc_self = Arc::new(self); @@ -1246,7 +1246,7 @@ where if new_durable > prev && let Some(ref tx) = self.log_flush_tx { - let _ = tx.send(crate::RoleEvent::LogFlushed { + let _ = tx.send(crate::InternalEvent::LogFlushed { durable_index: new_durable, }); } diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs index 1c4cc430..5e37ef0d 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs @@ -207,7 +207,7 @@ async fn test_term_index_functions_with_purged_logs() { /// Sequential multi-term insertion correctness test. /// /// Production invariant: all writes to BufferedRaftLog go through the single -/// Raft event-loop task; there are no concurrent writers. The previous version +/// inbound event-loop task; there are no concurrent writers. The previous version /// of this test spawned multiple tasks writing concurrently, which is not a /// production scenario and masked the real invariant. This test verifies /// first/last index tracking across five consecutive term segments. diff --git a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs index e01be03d..9bd74dde 100644 --- a/d-engine-core/src/test_utils/mock/mock_raft_builder.rs +++ b/d-engine-core/src/test_utils/mock/mock_raft_builder.rs @@ -9,6 +9,8 @@ use tokio::sync::watch; use tracing::trace; use crate::ElectionConfig; +use crate::InboundEvent; +use crate::InternalEvent; use crate::MockElectionCore; use crate::MockMembership; use crate::MockPurgeExecutor; @@ -22,12 +24,10 @@ use crate::Raft; use crate::RaftConfig; use crate::RaftContext; use crate::RaftCoreHandlers; -use crate::RaftEvent; use crate::RaftLog; use crate::RaftNodeConfig; use crate::RaftRole; use crate::RaftStorageHandles; -use crate::RoleEvent; use crate::SignalParams; use crate::StateMachine; use crate::follower_state::FollowerState; @@ -47,11 +47,11 @@ pub struct MockBuilder { pub turn_on_election: Option, shutdown_signal: watch::Receiver<()>, - pub(crate) event_tx: Option>, - pub(crate) event_rx: Option>, + pub(crate) event_tx: Option>, + pub(crate) event_rx: Option>, - pub(crate) role_tx: Option>, - pub(crate) role_rx: Option>, + pub(crate) internal_event_tx: Option>, + pub(crate) internal_event_rx: Option>, } impl MockBuilder { @@ -71,8 +71,8 @@ impl MockBuilder { turn_on_election: None, shutdown_signal, - role_tx: None, - role_rx: None, + internal_event_tx: None, + internal_event_rx: None, event_tx: None, event_rx: None, } @@ -121,7 +121,7 @@ impl MockBuilder { } pub fn build_raft(self) -> Raft { - let (role_tx, role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); let (event_tx, event_rx) = mpsc::channel(10); let (cmd_tx, cmd_rx) = mpsc::channel(1024); let ( @@ -135,8 +135,8 @@ impl MockBuilder { membership, purge_executor, node_config, - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, event_tx, event_rx, ) = ( @@ -156,8 +156,8 @@ impl MockBuilder { .validate() .expect("Should succeed to validate RaftNodeConfig") }), - self.role_tx.unwrap_or(role_tx), - self.role_rx.unwrap_or(role_rx), + self.internal_event_tx.unwrap_or(internal_event_tx), + self.internal_event_rx.unwrap_or(internal_event_rx), self.event_tx.unwrap_or(event_tx), self.event_rx.unwrap_or(event_rx), ); @@ -205,8 +205,8 @@ impl MockBuilder { }, membership, SignalParams { - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, event_tx, event_rx, cmd_tx, diff --git a/d-engine-server/src/api/embedded_client.rs b/d-engine-server/src/api/embedded_client.rs index c9bf3c56..0f0ffd76 100644 --- a/d-engine-server/src/api/embedded_client.rs +++ b/d-engine-server/src/api/embedded_client.rs @@ -20,8 +20,8 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; +use d_engine_core::InboundEvent; use d_engine_core::MaybeCloneOneshot; -use d_engine_core::RaftEvent; use d_engine_core::RaftOneshot; use d_engine_core::ScanResult; use d_engine_core::TypeConfig; @@ -49,7 +49,7 @@ use d_engine_core::watch::WatchRegistry; /// /// For standalone/gRPC mode use `GrpcClient` instead. Both implement `ClientApi`. pub struct EmbeddedClient { - event_tx: mpsc::Sender, + event_tx: mpsc::Sender, read_handle: EmbeddedReadHandle, client_id: u32, timeout: Duration, @@ -73,7 +73,7 @@ impl Clone for EmbeddedClient { impl EmbeddedClient { /// Internal constructor (used by EmbeddedEngine). pub(crate) fn new_internal( - event_tx: mpsc::Sender, + event_tx: mpsc::Sender, read_handle: EmbeddedReadHandle, client_id: u32, timeout: Duration, @@ -531,7 +531,7 @@ impl EmbeddedClient { let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::ClusterConf(request, resp_tx)) + .send(InboundEvent::ClusterConf(request, resp_tx)) .await .map_err(|_| channel_closed_error())?; diff --git a/d-engine-server/src/membership/raft_membership.rs b/d-engine-server/src/membership/raft_membership.rs index 408ed2cf..672d2ef8 100644 --- a/d-engine-server/src/membership/raft_membership.rs +++ b/d-engine-server/src/membership/raft_membership.rs @@ -759,7 +759,7 @@ impl RaftMembership where T: TypeConfig, { - /// Returns `(Self, zombie_rx)`. Caller must pass `zombie_rx` to the Raft event loop + /// Returns `(Self, zombie_rx)`. Caller must pass `zombie_rx` to the inbound event loop /// so that `ZombieDetected` signals reach the leader. /// /// Call [`subscribe_membership`] on the returned `Self` to obtain a @@ -820,7 +820,7 @@ where /// Returns true if the zombie signal for `node_id` should still be acted on. /// /// Delegates to the inner health monitor. The bridge task calls this before - /// forwarding a `ZombieDetected` signal to the Raft event loop so that a + /// forwarding a `ZombieDetected` signal to the inbound event loop so that a /// stale signal (peer already recovered via `record_success`) is dropped. pub(crate) fn is_zombie_valid( &self, diff --git a/d-engine-server/src/network/grpc/grpc_raft_service.rs b/d-engine-server/src/network/grpc/grpc_raft_service.rs index 75236e7d..717d97b3 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service.rs @@ -4,9 +4,9 @@ use crate::Node; use crate::proto_convert; +use d_engine_core::InboundEvent; use d_engine_core::MaybeCloneOneshot; use d_engine_core::MaybeCloneOneshotReceiver; -use d_engine_core::RaftEvent; use d_engine_core::RaftOneshot; use d_engine_core::TypeConfig; #[cfg(feature = "watch")] @@ -85,7 +85,10 @@ where let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::ReceiveVoteRequest(request.into_inner(), resp_tx)) + .send(InboundEvent::ReceiveVoteRequest( + request.into_inner(), + resp_tx, + )) .await .map_err(|_| Status::internal("Event channel closed"))?; let timeout_duration = @@ -116,7 +119,7 @@ where let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::AppendEntries( + .send(InboundEvent::AppendEntries( request.into_inner(), vec![resp_tx], )) @@ -135,7 +138,7 @@ where /// Processes a persistent bidirectional AppendEntries stream from the cluster leader. /// /// Decouples request ingestion from response emission: - /// - recv task: reads batches from the stream, dispatches each as a `RaftEvent::AppendEntries` + /// - recv task: reads batches from the stream, dispatches each as a `InboundEvent::AppendEntries` /// (non-blocking between batches) /// - forwarder task: drains ordered response handles sequentially; ordering is guaranteed /// by the Raft single-threaded event loop @@ -180,7 +183,7 @@ where match result { Some(Ok(req)) => { let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); - if event_tx.send(RaftEvent::AppendEntries(req, vec![resp_tx])).await.is_err() { + if event_tx.send(InboundEvent::AppendEntries(req, vec![resp_tx])).await.is_err() { warn!("[stream_append_entries|recv] event_tx closed"); break; } @@ -255,7 +258,7 @@ where let (startup_tx, startup_rx) = oneshot::channel::>(); self.event_tx - .send(RaftEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx)) + .send(InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx)) .await .map_err(|_| Status::internal("Event channel closed"))?; @@ -301,7 +304,7 @@ where }); self.event_tx - .send(RaftEvent::InstallSnapshotChunk(rx, resp_tx)) + .send(InboundEvent::InstallSnapshotChunk(rx, resp_tx)) .await .map_err(|_| Status::internal("Event channel closed"))?; @@ -334,7 +337,10 @@ where let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::ClusterConfUpdate(request.into_inner(), resp_tx)) + .send(InboundEvent::ClusterConfUpdate( + request.into_inner(), + resp_tx, + )) .await .map_err(|_| Status::internal("Event channel closed"))?; @@ -361,7 +367,7 @@ where let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::ClusterConf(request.into_inner(), resp_tx)) + .send(InboundEvent::ClusterConf(request.into_inner(), resp_tx)) .await .map_err(|_| Status::internal("Event channel closed"))?; @@ -383,7 +389,7 @@ where let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::JoinCluster(request.into_inner(), resp_tx)) + .send(InboundEvent::JoinCluster(request.into_inner(), resp_tx)) .await .map_err(|_| Status::internal("Event channel closed"))?; @@ -404,7 +410,7 @@ where let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); self.event_tx - .send(RaftEvent::DiscoverLeader(request.into_inner(), resp_tx)) + .send(InboundEvent::DiscoverLeader(request.into_inner(), resp_tx)) .await .map_err(|_| Status::internal("Event channel closed"))?; diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs index 987488b6..20edc7f0 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs @@ -3,13 +3,13 @@ use std::time::Duration; use crate::ApplyResult; use d_engine_core::AppendResponseWithUpdates; +use d_engine_core::InternalEvent; use d_engine_core::MockElectionCore; use d_engine_core::MockMembership; use d_engine_core::MockRaftLog; use d_engine_core::MockReplicationCore; use d_engine_core::MockTypeConfig; use d_engine_core::RaftNodeConfig; -use d_engine_core::RoleEvent; use d_engine_core::convert::safe_kv_bytes; use d_engine_proto::client::ClientReadRequest; use d_engine_proto::client::ClientWriteRequest; @@ -266,11 +266,11 @@ async fn test_handle_rpc_services_successfully() { // Create role channel so the test can inject LogFlushed events directly into the raft loop. // Commit in single-voter mode is now async (driven by LogFlushed from BufferedRaftLog); // since this test uses MockRaftLog (no real batch_processor), we send LogFlushed manually. - let (role_tx, role_rx) = mpsc::unbounded_channel::(); - let test_role_tx = role_tx.clone(); + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel::(); + let test_internal_event_tx = internal_event_tx.clone(); let mut builder = MockBuilder::new(graceful_rx); - builder.role_tx = Some(role_tx); - builder.role_rx = Some(role_rx); + builder.internal_event_tx = Some(internal_event_tx); + builder.internal_event_rx = Some(internal_event_rx); let node = builder .with_raft_log(raft_log) .with_membership(membership) @@ -322,15 +322,15 @@ async fn test_handle_rpc_services_successfully() { // Advance commit_index via LogFlushed — moves entries from // pending_client_writes into pending_write_apply (drain_pending_client_writes). let flush_idx = li_flush.load(Ordering::Relaxed); - let _ = test_role_tx.send(RoleEvent::LogFlushed { + let _ = test_internal_event_tx.send(InternalEvent::LogFlushed { durable_index: flush_idx, }); // Yield again so the raft loop processes LogFlushed before ApplyCompleted. for _ in 0..5 { tokio::task::yield_now().await; } - // ApplyCompleted is now sent via role_tx (P2) to avoid priority inversion. - let _ = test_role_tx.send(RoleEvent::ApplyCompleted { + // ApplyCompleted is now sent via internal_event_tx (P2) to avoid priority inversion. + let _ = test_internal_event_tx.send(InternalEvent::ApplyCompleted { last_index: 2, results: vec![ApplyResult { index: 2, diff --git a/d-engine-server/src/network/health_monitor.rs b/d-engine-server/src/network/health_monitor.rs index 0c7a4307..e6cbabc1 100644 --- a/d-engine-server/src/network/health_monitor.rs +++ b/d-engine-server/src/network/health_monitor.rs @@ -21,13 +21,13 @@ pub(crate) struct RaftHealthMonitor { pub(crate) failure_counts: DashMap, pub(crate) zombie_threshold: u32, /// Fires node_id when failure_count first reaches zombie_threshold. - /// Consumed by the Raft event loop (core layer) via select!. + /// Consumed by the inbound event loop (core layer) via select!. /// Server layer holds only Sender — zero dependency on core types. zombie_tx: mpsc::Sender, } impl RaftHealthMonitor { - /// Returns (monitor, zombie_rx). Caller passes zombie_rx to the Raft event loop. + /// Returns (monitor, zombie_rx). Caller passes zombie_rx to the inbound event loop. pub(crate) fn new(zombie_threshold: u32) -> (Self, mpsc::Receiver) { let (zombie_tx, zombie_rx) = mpsc::channel(64); ( @@ -80,7 +80,7 @@ impl RaftHealthMonitor { /// /// A zombie is invalid once `record_success` has been called (peer recovered), /// which removes the entry from `failure_counts`. The bridge task uses this - /// to drop stale zombie signals before forwarding them to the Raft event loop. + /// to drop stale zombie signals before forwarding them to the inbound event loop. pub(crate) fn is_zombie_valid( &self, node_id: u32, diff --git a/d-engine-server/src/network/health_monitor_test.rs b/d-engine-server/src/network/health_monitor_test.rs index d16ffd69..92f8d338 100644 --- a/d-engine-server/src/network/health_monitor_test.rs +++ b/d-engine-server/src/network/health_monitor_test.rs @@ -109,7 +109,7 @@ async fn test_zombie_signal_re_triggers_after_signal_lost() { // Bug 2: zombie signal must be invalidated when the peer recovers // // Scenario: ZombieDetected was queued in the channel, but before the bridge -// task forwards it to the Raft event loop the peer reconnects successfully. +// task forwards it to the inbound event loop the peer reconnects successfully. // The bridge task calls is_zombie_valid() to avoid proposing BatchRemove for a // healthy node. // diff --git a/d-engine-server/src/node/builder.rs b/d-engine-server/src/node/builder.rs index 220f59aa..0bdc1e90 100644 --- a/d-engine-server/src/node/builder.rs +++ b/d-engine-server/src/node/builder.rs @@ -40,6 +40,7 @@ use d_engine_core::DefaultCommitHandler; use d_engine_core::DefaultPurgeExecutor; use d_engine_core::DefaultStateMachineHandler; use d_engine_core::ElectionHandler; +use d_engine_core::InternalEvent; use d_engine_core::LogSizePolicy; use d_engine_core::NewCommitData; use d_engine_core::Raft; @@ -51,7 +52,6 @@ use d_engine_core::RaftRole; use d_engine_core::RaftStorageHandles; use d_engine_core::ReplicationHandler; use d_engine_core::Result; -use d_engine_core::RoleEvent; use d_engine_core::SignalParams; use d_engine_core::StateMachine; use d_engine_core::StateMachineWorker; @@ -305,8 +305,8 @@ where info!("Node startup, Last applied index: {}", last_applied_index); // Create role channel before raft_log so log_flush_tx can be passed to start(). - // role_rx is passed to Raft::new() below; only the creation order changes. - let (role_tx, role_rx) = mpsc::unbounded_channel(); + // internal_event_rx is passed to Raft::new() below; only the creation order changes. + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); let raft_log = { let (log, receiver) = BufferedRaftLog::new( @@ -316,8 +316,8 @@ where ); // Start processor and get Arc-wrapped instance. - // Pass role_tx so batch_processor sends RoleEvent::LogFlushed after each fsync. - log.start(receiver, Some(role_tx.clone())) + // Pass internal_event_tx so batch_processor sends InternalEvent::LogFlushed after each fsync. + log.start(receiver, Some(internal_event_tx.clone())) }; // Peer health channels: transport fires try_send(peer_id) on stream failure/success. @@ -424,24 +424,25 @@ where let purge_executor = DefaultPurgeExecutor::new(raft_log.clone()); - // role_tx / role_rx created earlier (before raft_log) to pass log_flush_tx to start(). + // internal_event_tx / internal_event_rx created earlier (before raft_log) to pass log_flush_tx to start(). let (event_tx, event_rx) = mpsc::channel(10240); let (cmd_tx, cmd_rx) = mpsc::channel(node_config.raft.cmd_channel_capacity); - let role_tx_clone = role_tx.clone(); - let role_tx_for_sm = role_tx.clone(); // used in SM worker (ApplyCompleted → P2) + let internal_event_tx_clone = internal_event_tx.clone(); + let internal_event_tx_for_sm = internal_event_tx.clone(); // used in SM worker (ApplyCompleted → P2) - // Bridge zombie signals from health monitor → role event loop. + // Bridge zombie signals from health monitor → internal event loop. // Exits naturally when zombie_tx drops with RaftMembership (channel closed → recv None). // Validates each signal against the health monitor before forwarding: if the peer // recovered (record_success called) after the signal was queued, drop the stale signal // to prevent BatchRemove for a healthy node. - let role_tx_for_zombie = role_tx.clone(); + let internal_event_tx_for_zombie = internal_event_tx.clone(); let membership_for_zombie = Arc::clone(&membership); tokio::spawn(async move { let mut zombie_rx = zombie_rx; while let Some(node_id) = zombie_rx.recv().await { if membership_for_zombie.is_zombie_valid(node_id) { - let _ = role_tx_for_zombie.send(RoleEvent::ZombieDetected(node_id)); + let _ = + internal_event_tx_for_zombie.send(InternalEvent::ZombieDetected(node_id)); } } }); @@ -512,8 +513,8 @@ where }, membership.clone(), SignalParams::new( - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, event_tx, event_rx, cmd_tx.clone(), @@ -538,7 +539,7 @@ where node_id, state_machine_handler.clone(), sm_apply_rx, - role_tx_for_sm, + internal_event_tx_for_sm, self.shutdown_signal.clone(), ); let sm_worker_handle = Self::spawn_state_machine_worker(sm_worker); @@ -548,7 +549,7 @@ where state_machine_handler, raft_log: raft_core.ctx.storage.raft_log.clone(), membership: membership.clone(), - role_tx: role_tx_clone, + internal_event_tx: internal_event_tx_clone, sm_apply_tx, shutdown_signal, max_batch_size: raft_core.ctx.node_config.raft.batching.max_batch_size, diff --git a/d-engine-server/src/node/mod.rs b/d-engine-server/src/node/mod.rs index 4bd53766..0f5a1189 100644 --- a/d-engine-server/src/node/mod.rs +++ b/d-engine-server/src/node/mod.rs @@ -42,9 +42,9 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; +use d_engine_core::InboundEvent; use d_engine_core::Membership; use d_engine_core::Raft; -use d_engine_core::RaftEvent; use d_engine_core::RaftNodeConfig; use d_engine_core::ReadLease; use d_engine_core::Result; @@ -81,7 +81,7 @@ where // Network & Storage events, (copied from Raft) // TODO: find a better solution - pub(crate) event_tx: mpsc::Sender, + pub(crate) event_tx: mpsc::Sender, // Client commands (drain-driven) pub(crate) cmd_tx: mpsc::Sender, @@ -156,7 +156,7 @@ where /// - **Learner**: Skip cluster ready check, join cluster after warmup /// - **Voter**: Wait for cluster ready, then warmup connections /// - /// Both paths converge to the Raft event processing loop. + /// Both paths converge to the inbound event processing loop. /// /// # Errors /// Returns `Err` if any bootstrap step or Raft execution fails. diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs index 39f9158b..00961385 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_state_machine.rs @@ -351,7 +351,7 @@ impl RocksDBStateMachine { /// Piggyback cleanup: Remove expired keys with time budget /// /// This method is called during apply_chunk to cleanup expired keys - /// opportunistically (piggyback on existing Raft events). + /// opportunistically (piggyback on existing inbound events). /// /// # Arguments /// * `max_duration_ms` - Maximum time budget for cleanup (milliseconds) @@ -988,7 +988,7 @@ impl StateMachine for RocksDBStateMachine { // // All blocking RocksDB operations (create_dir_all, flush_cf_opt, export_column_family) // run on tokio's blocking thread pool via spawn_blocking, not on an async worker thread. - // This prevents snapshot disk I/O from starving the Raft event loop under concurrent + // This prevents snapshot disk I/O from starving the inbound event loop under concurrent // writes (#315). let db = self.load_db()?; // Arc: Send, safe to move into spawn_blocking let dir = new_snapshot_dir.clone(); diff --git a/d-engine-server/src/test_utils/mock/mock_node_builder.rs b/d-engine-server/src/test_utils/mock/mock_node_builder.rs index 8cbeb150..0adec374 100644 --- a/d-engine-server/src/test_utils/mock/mock_node_builder.rs +++ b/d-engine-server/src/test_utils/mock/mock_node_builder.rs @@ -6,6 +6,8 @@ use std::sync::atomic::AtomicBool; use bytes::Bytes; use d_engine_core::ElectionConfig; +use d_engine_core::InboundEvent; +use d_engine_core::InternalEvent; use d_engine_core::MockElectionCore; use d_engine_core::MockMembership; use d_engine_core::MockPurgeExecutor; @@ -18,12 +20,10 @@ use d_engine_core::Raft; use d_engine_core::RaftConfig; use d_engine_core::RaftContext; use d_engine_core::RaftCoreHandlers; -use d_engine_core::RaftEvent; use d_engine_core::RaftLog; use d_engine_core::RaftNodeConfig; use d_engine_core::RaftRole; use d_engine_core::RaftStorageHandles; -use d_engine_core::RoleEvent; use d_engine_core::SignalParams; use d_engine_core::StateMachine; use d_engine_core::follower_state::FollowerState; @@ -118,14 +118,14 @@ pub struct MockBuilder { shutdown_signal: watch::Receiver<()>, /// Internal event channel sender - pub(crate) event_tx: Option>, + pub(crate) event_tx: Option>, /// Internal event channel receiver - pub(crate) event_rx: Option>, + pub(crate) event_rx: Option>, /// Internal role change channel sender - pub(crate) role_tx: Option>, + pub(crate) internal_event_tx: Option>, /// Internal role change channel receiver - pub(crate) role_rx: Option>, + pub(crate) internal_event_rx: Option>, } impl MockBuilder { @@ -153,8 +153,8 @@ impl MockBuilder { turn_on_election: None, shutdown_signal, - role_tx: None, - role_rx: None, + internal_event_tx: None, + internal_event_rx: None, event_tx: None, event_rx: None, } @@ -210,7 +210,7 @@ impl MockBuilder { /// This is the most commonly used build method. Returns a fully /// initialized Raft instance ready for testing. pub fn build_raft(self) -> Raft { - let (role_tx, role_rx) = mpsc::unbounded_channel(); + let (internal_event_tx, internal_event_rx) = mpsc::unbounded_channel(); let (event_tx, event_rx) = mpsc::channel(10); let (cmd_tx, cmd_rx) = mpsc::channel(1024); let ( @@ -224,8 +224,8 @@ impl MockBuilder { membership, purge_executor, node_config, - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, event_tx, event_rx, ) = ( @@ -245,8 +245,8 @@ impl MockBuilder { .validate() .expect("Should succeed to validate RaftNodeConfig") }), - self.role_tx.unwrap_or(role_tx), - self.role_rx.unwrap_or(role_rx), + self.internal_event_tx.unwrap_or(internal_event_tx), + self.internal_event_rx.unwrap_or(internal_event_rx), self.event_tx.unwrap_or(event_tx), self.event_rx.unwrap_or(event_rx), ); @@ -294,8 +294,8 @@ impl MockBuilder { }, membership, SignalParams::new( - role_tx, - role_rx, + internal_event_tx, + internal_event_rx, event_tx, event_rx, cmd_tx, diff --git a/d-engine-server/tests/drain_batching/select_fairness_embedded.rs b/d-engine-server/tests/drain_batching/select_fairness_embedded.rs index 8b928d83..9d2b11a3 100644 --- a/d-engine-server/tests/drain_batching/select_fairness_embedded.rs +++ b/d-engine-server/tests/drain_batching/select_fairness_embedded.rs @@ -68,7 +68,7 @@ max_batch_size = 100 /// /// **Assertions**: /// - Each drain processes max_batch_size (100), then returns to select -/// - Internal Raft events processed within reasonable time (<50ms p99) +/// - Internal inbound events processed within reasonable time (<50ms p99) /// - No event starvation (cluster remains healthy) /// - System remains responsive to protocol messages /// @@ -138,7 +138,7 @@ async fn test_select_fairness_drain_no_starvation() { let start = Instant::now(); // Linearizable read requires verify_leadership - // This tests if P4 branch (raft events) is responsive + // This tests if P4 branch (inbound events) is responsive let _result = monitor_client.get_linearizable(b"monitor_key").await; let latency = start.elapsed(); diff --git a/d-engine-server/tests/embedded_client/embedded_client_operations.rs b/d-engine-server/tests/embedded_client/embedded_client_operations.rs index 83c02451..5e4a513d 100644 --- a/d-engine-server/tests/embedded_client/embedded_client_operations.rs +++ b/d-engine-server/tests/embedded_client/embedded_client_operations.rs @@ -568,7 +568,7 @@ async fn test_put_with_ttl_readable_immediately() { /// put_with_ttl must succeed when no [raft.state_machine.lease] section is present /// in the config (i.e. default/zero-config). Previously this caused a fatal crash: /// the state machine returned an error which the SM Worker mapped to -/// RoleEvent::FatalError, shutting down the entire node. +/// InternalEvent::FatalError, shutting down the entire node. /// /// Verification steps: /// 1. Engine starts with default config (no lease section) diff --git a/d-engine/src/docs/server_guide/snapshot-guarantees.md b/d-engine/src/docs/server_guide/snapshot-guarantees.md index 909f4cf1..ec3da4b6 100644 --- a/d-engine/src/docs/server_guide/snapshot-guarantees.md +++ b/d-engine/src/docs/server_guide/snapshot-guarantees.md @@ -7,7 +7,7 @@ and configuration recommendations for production deployments. **Write operations are never blocked by snapshot generation.** Snapshot creation runs in a background `tokio::spawn` task, isolating compression I/O -from the Raft event loop. Writes continue uninterrupted while a snapshot is being +from the Inbound event loop. Writes continue uninterrupted while a snapshot is being compressed and written to disk. **Committed data is never lost during interrupted transfers.** @@ -65,15 +65,15 @@ disk after a new one is created. Keep at least 2 for rollback and debugging head ## Configuration Reference -| Field | Default | Description | -|---|---|---| -| `max_log_entries_before_snapshot` | 1000 | Log entries before snapshot triggers | -| `retained_log_entries` | 1 | Log entries to retain after snapshot | -| `chunk_size` | 1024 (1 KB) | Size of each transfer chunk in bytes | -| `receive_chunk_timeout_in_sec` | 30 | Per-chunk receive timeout on follower | -| `transfer_timeout_in_sec` | 600 | Overall transfer timeout | -| `max_bandwidth_mbps` | 1 | Transfer bandwidth cap (Mbps) | -| `cleanup_retain_count` | 2 | Number of past snapshot files to keep | +| Field | Default | Description | +| --------------------------------- | ----------- | ------------------------------------- | +| `max_log_entries_before_snapshot` | 1000 | Log entries before snapshot triggers | +| `retained_log_entries` | 1 | Log entries to retain after snapshot | +| `chunk_size` | 1024 (1 KB) | Size of each transfer chunk in bytes | +| `receive_chunk_timeout_in_sec` | 30 | Per-chunk receive timeout on follower | +| `transfer_timeout_in_sec` | 600 | Overall transfer timeout | +| `max_bandwidth_mbps` | 1 | Transfer bandwidth cap (Mbps) | +| `cleanup_retain_count` | 2 | Number of past snapshot files to keep | ## Further Reading From 030b3d00ffe7eea82a8919824859d4fc5fac5bbe Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:44:31 +0800 Subject: [PATCH 3/4] fix #413: Follower DiscoverLeader redirect + Candidate stale-event dispatch tests --- .../src/raft_role/candidate_state.rs | 7 +- d-engine-core/src/raft_role/follower_state.rs | 25 +++- .../src/raft_role/follower_state_test.rs | 140 +++++++++++++++--- .../pending_commit_actions_test.rs | 2 +- d-engine-core/src/raft_role/learner_state.rs | 2 +- d-engine-core/src/raft_role/role_state.rs | 4 +- d-engine-core/src/raft_test/mod.rs | 4 + .../src/raft_test/raft_comprehensive_tests.rs | 107 +++++++++++++ .../src/membership/raft_membership.rs | 4 +- 9 files changed, 257 insertions(+), 38 deletions(-) diff --git a/d-engine-core/src/raft_role/candidate_state.rs b/d-engine-core/src/raft_role/candidate_state.rs index 30df506a..32788abe 100644 --- a/d-engine-core/src/raft_role/candidate_state.rs +++ b/d-engine-core/src/raft_role/candidate_state.rs @@ -1,13 +1,12 @@ -use std::fmt::Debug; -use std::marker::PhantomData; -use std::sync::Arc; - use async_trait::async_trait; use d_engine_proto::common::LogId; use d_engine_proto::common::NodeRole::Candidate; use d_engine_proto::server::cluster::ClusterConfUpdateResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::election::VotedFor; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::sync::Arc; use tokio::sync::mpsc; use tokio::time::Instant; use tonic::Status; diff --git a/d-engine-core/src/raft_role/follower_state.rs b/d-engine-core/src/raft_role/follower_state.rs index 815e93f2..6670b4c7 100644 --- a/d-engine-core/src/raft_role/follower_state.rs +++ b/d-engine-core/src/raft_role/follower_state.rs @@ -2,6 +2,7 @@ use async_trait::async_trait; use d_engine_proto::common::LogId; use d_engine_proto::common::NodeRole::Follower; use d_engine_proto::server::cluster::ClusterConfUpdateResponse; +use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::storage::SnapshotAck; use d_engine_proto::server::storage::SnapshotMetadata; @@ -411,14 +412,22 @@ impl RaftRoleState for FollowerState { InboundEvent::DiscoverLeader(request, sender) => { debug!(?request, "Follower::InboundEvent::DiscoverLeader"); - sender - .send(Err(Status::permission_denied( - "Follower should not response DiscoverLeader event.", - ))) - .map_err(|e| { - error!("Failed to send: {:?}", e); - NetworkError::SingalSendFailed(format!("{:?}", e)) - })?; + let response = match self.shared_state().current_leader() { + Some(leader_id) => match ctx.membership().retrieve_node_meta(leader_id).await { + Some(meta) => Ok(LeaderDiscoveryResponse { + leader_id, + leader_address: meta.address, + term: self.current_term(), + }), + None => Err(Status::not_found("Leader metadata not found")), + }, + None => Err(Status::unavailable("Leader not discovered")), + }; + + sender.send(response).map_err(|e| { + error!("Failed to send: {:?}", e); + NetworkError::SingalSendFailed(format!("{:?}", e)) + })?; return Ok(()); } diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index e701278c..9cd0b4f3 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -1348,28 +1348,25 @@ async fn test_handle_join_cluster_rejects_on_follower() { ); } -/// Test: FollowerState rejects LeaderDiscovery request +/// Test: Follower returns Unavailable when it has not yet learned the leader. /// /// Scenario: -/// - Follower receives LeaderDiscoveryRequest (client wants to find leader) -/// - Follower doesn't know current leader or is not the leader +/// - Follower has no current_leader (e.g. node just started, no heartbeat received yet) +/// - Client sends DiscoverLeader /// /// Expected: -/// - Returns Status error with Code::PermissionDenied /// - handle_inbound_event returns Ok() (not a fatal error) -/// - No state changes -/// -/// This validates that follower correctly rejects leader discovery -/// when it cannot provide leader information. +/// - Response carries Status::Unavailable — tells the client to retry later /// /// Original: test_handle_inbound_event_case11 #[tokio::test] -async fn test_handle_leader_discovery_rejects_on_follower() { +async fn test_discover_leader_returns_unavailable_when_leader_unknown() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); let mut state = FollowerState::::new(1, context.node_config.clone(), None, None); + // No set_current_leader call — follower has no leader info. let request = LeaderDiscoveryRequest { node_id: 2, @@ -1379,23 +1376,126 @@ async fn test_handle_leader_discovery_rejects_on_follower() { let inbound_event = InboundEvent::DiscoverLeader(request, resp_tx); let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - // Action: Handle DiscoverLeader event let result = state.handle_inbound_event(inbound_event, &context, internal_event_tx).await; - // Verify: Returns Ok (not a fatal error) - assert!( - result.is_ok(), - "handle_inbound_event should return Ok for DiscoverLeader" + assert!(result.is_ok(), "handle_inbound_event must not be fatal"); + + let response = resp_rx.recv().await.expect("Should receive response"); + assert!(response.is_err()); + assert_eq!( + response.unwrap_err().code(), + Code::Unavailable, + "Unknown leader → Unavailable (client should retry)" ); +} - // Verify: Response with PermissionDenied +/// Test: Follower redirects client to known leader address. +/// +/// Scenario: +/// - Follower received a heartbeat from node 3 and stored it as current_leader +/// - Client sends DiscoverLeader +/// - Membership returns node 3's address +/// +/// Expected: +/// - Response carries leader_id=3, leader_address, and current term +/// +/// Note: The returned leader_id may be stale if the leader stepped down after the last heartbeat. +/// This is acceptable — the client will get a NotLeader error from that node and retry. +/// The term returned alongside helps the client detect staleness. +#[tokio::test] +async fn test_discover_leader_returns_known_leader_address() { + use d_engine_proto::common::{NodeRole::Leader, NodeStatus}; + use d_engine_proto::server::cluster::NodeMeta; + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let mut membership = MockMembership::new(); + membership + .expect_retrieve_node_meta() + .with(mockall::predicate::eq(3u32)) + .returning(|_| { + Some(NodeMeta { + id: 3, + address: "10.0.0.3:50051".to_string(), + role: Leader.into(), + status: NodeStatus::Active.into(), + }) + }); + context.membership = Arc::new(membership); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state.set_current_leader(3); + state.update_current_term(5); + + let request = LeaderDiscoveryRequest { + node_id: 2, + requester_address: "127.0.0.1:9090".to_string(), + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::DiscoverLeader(request, resp_tx), + &context, + internal_event_tx, + ) + .await; + + assert!(result.is_ok()); + let response = resp_rx.recv().await.expect("Should receive response").unwrap(); + assert_eq!(response.leader_id, 3); + assert_eq!(response.leader_address, "10.0.0.3:50051"); + assert_eq!( + response.term, 5, + "Response must carry current term for staleness detection" + ); +} + +/// Test: Follower knows a leader ID but cannot find its metadata — returns NotFound. +/// +/// Scenario: +/// - Follower has current_leader=3 but membership has no metadata for node 3 +/// (e.g. membership config not yet propagated after cluster reconfiguration) +/// +/// Expected: +/// - Response carries Status::NotFound +#[tokio::test] +async fn test_discover_leader_returns_not_found_when_metadata_missing() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let mut membership = MockMembership::new(); + membership.expect_retrieve_node_meta().returning(|_| None); + context.membership = Arc::new(membership); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state.set_current_leader(3); + + let request = LeaderDiscoveryRequest { + node_id: 2, + requester_address: "127.0.0.1:9090".to_string(), + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::DiscoverLeader(request, resp_tx), + &context, + internal_event_tx, + ) + .await; + + assert!(result.is_ok()); let response = resp_rx.recv().await.expect("Should receive response"); - assert!(response.is_err(), "Response should be error"); - let status = response.unwrap_err(); assert_eq!( - status.code(), - Code::PermissionDenied, - "Should return PermissionDenied" + response.unwrap_err().code(), + Code::NotFound, + "Known leader ID but missing metadata → NotFound" ); } diff --git a/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs b/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs index dc41a434..7414e7af 100644 --- a/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/pending_commit_actions_test.rs @@ -6,7 +6,7 @@ //! //! Three key behaviors: //! 1. `drain_commit_actions` fires `InternalEvent::NoopCommitted` when a `LeaderNoop` entry commits. -//! 2. `drain_commit_actions` fires `InternalEvent::JoinCommitted` when a `NodeJoin` entry commits. +//! 2. `drain_commit_actions` delivers `NodeJoin` completion directly to the join response sender //! (no `InternalEvent` is emitted for NodeJoin). //! 3. tick() sends `InternalEvent::BecomeFollower` for expired noop entries (leadership lost). use std::time::Duration; diff --git a/d-engine-core/src/raft_role/learner_state.rs b/d-engine-core/src/raft_role/learner_state.rs index 0e580aed..39200590 100644 --- a/d-engine-core/src/raft_role/learner_state.rs +++ b/d-engine-core/src/raft_role/learner_state.rs @@ -548,7 +548,7 @@ impl RaftRoleState for LearnerState { let result = state_machine_handler.create_snapshot().await; info!("SnapshotCreated event will be processed in another event thread"); if let Err(e) = internal_event_tx.send(InternalEvent::SnapshotCreated(result)) { - error!("Follower failed to send snapshot creation result: {}", e); + error!("Learner failed to send snapshot creation result: {}", e); } }); diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index 3a6b7d46..0d156acb 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -649,7 +649,7 @@ pub trait RaftRoleState: Send + Sync + 'static { current_role: "None Leader", required_role: "Leader", context: format!( - "None Leader node {} receives InboundEvent::PromoteReadyLearners", + "None Leader node {} receives InternalEvent::PromoteReadyLearners", ctx.node_id ), } @@ -686,7 +686,7 @@ pub trait RaftRoleState: Send + Sync + 'static { current_role: "None Leader/Learner", required_role: "Leader/Learner", context: format!( - "None Leader/Learner node {} receives InboundEvent::PromoteReadyLearners", + "None Leader/Learner node {} receives InternalEvent::PromoteReadyLearners", ctx.node_id ), } diff --git a/d-engine-core/src/raft_test/mod.rs b/d-engine-core/src/raft_test/mod.rs index 3f983a82..e232f021 100644 --- a/d-engine-core/src/raft_test/mod.rs +++ b/d-engine-core/src/raft_test/mod.rs @@ -104,18 +104,22 @@ //! //! #### B9. SnapshotCreated (migrated from P4 event_tx → P2 internal_event_tx) //! - **B9.1**: Follower + Err payload — dispatch returns Ok (non-fatal error swallowed) +//! - **B9.2**: Candidate — RoleViolation non-fatal, swallowed by dispatch layer //! //! #### B10. StepDownSelfRemoved (migrated from P4 event_tx → P2 internal_event_tx) //! - **B10.1**: Leader — handle_self_removed enqueues BecomeFollower; role transitions to Follower //! //! #### B11. MembershipApplied (migrated from P4 event_tx → P2 internal_event_tx) //! - **B11.1**: Follower — RoleViolation swallowed; leader-specific logic in membership_change_test +//! - **B11.2**: Candidate — RoleViolation non-fatal, swallowed by dispatch layer //! //! #### B12. PromoteReadyLearners (migrated from P4 event_tx → P2 internal_event_tx) //! - **B12.1**: Follower — RoleViolation swallowed; leader-specific logic in membership_change_test +//! - **B12.2**: Candidate — RoleViolation non-fatal, swallowed by dispatch layer //! //! #### B13. LogPurgeCompleted (migrated from P4 event_tx → P2 internal_event_tx) //! - **B13.1**: Follower — dispatch returns Ok, last_purged_index updated +//! - **B13.2**: Candidate — RoleViolation non-fatal, swallowed by dispatch layer //! //! ### C. Leader Initialization Tests (Peer State Setup) //! diff --git a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs index cac344cd..778ea616 100644 --- a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs +++ b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs @@ -1225,6 +1225,113 @@ async fn test_log_purge_completed_dispatch_on_follower() { assert!(is_follower(raft.role.as_i32())); } +/// B14 — SnapshotCreated dispatched to Candidate +/// +/// Candidate has no snapshot state. The default handler returns RoleViolation (non-fatal). +/// The dispatch layer swallows it and returns Ok — role is unchanged. +/// +/// This guards against a regression where RoleViolation is reclassified as fatal, +/// which would cause the Raft process to exit on a stale in-flight event. +#[tokio::test] +async fn test_snapshot_created_dispatch_on_candidate() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + assert!(is_candidate(raft.role.as_i32())); + + let err_result = Err(crate::SnapshotError::OperationFailed("simulated".into()).into()); + raft.handle_internal_event(InternalEvent::SnapshotCreated(err_result)) + .await + .expect("SnapshotCreated on Candidate must return Ok (RoleViolation swallowed)"); + + assert!( + is_candidate(raft.role.as_i32()), + "Role must remain Candidate" + ); +} + +/// B15 — MembershipApplied dispatched to Candidate +/// +/// Candidate has no membership cache to refresh. The default handler returns RoleViolation +/// (non-fatal). The dispatch layer swallows it and returns Ok — role is unchanged. +/// +/// This guards against a regression where a stale MembershipApplied event arriving during +/// an election causes the Raft process to exit instead of being silently ignored. +#[tokio::test] +async fn test_membership_applied_dispatch_on_candidate() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + assert!(is_candidate(raft.role.as_i32())); + + raft.handle_internal_event(InternalEvent::MembershipApplied) + .await + .expect("MembershipApplied on Candidate must return Ok (RoleViolation swallowed)"); + + assert!( + is_candidate(raft.role.as_i32()), + "Role must remain Candidate" + ); +} + +/// B16 — PromoteReadyLearners dispatched to Candidate +/// +/// Candidate has no pending_promotions state. The default handler returns RoleViolation +/// (non-fatal). The dispatch layer swallows it and returns Ok — role is unchanged. +/// +/// This guards against a regression where a stale PromoteReadyLearners event arriving +/// during an election causes the Raft process to exit. +#[tokio::test] +async fn test_promote_ready_learners_dispatch_on_candidate() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + assert!(is_candidate(raft.role.as_i32())); + + raft.handle_internal_event(InternalEvent::PromoteReadyLearners) + .await + .expect("PromoteReadyLearners on Candidate must return Ok (RoleViolation swallowed)"); + + assert!( + is_candidate(raft.role.as_i32()), + "Role must remain Candidate" + ); +} + +/// B17 — LogPurgeCompleted dispatched to Candidate +/// +/// Candidate has no last_purged_index to update. The default handler returns RoleViolation +/// (non-fatal). The dispatch layer swallows it and returns Ok — role is unchanged. +/// +/// Candidate cannot receive stale Leader events in practice (role_tx is fully drained +/// between Leader→Follower and Follower→Candidate). This test pins the non-fatal +/// classification so a future refactor cannot silently break the invariant. +#[tokio::test] +async fn test_log_purge_completed_dispatch_on_candidate() { + use d_engine_proto::common::LogId; + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut raft = MockBuilder::new(graceful_rx).build_raft(); + + raft.handle_internal_event(InternalEvent::BecomeCandidate).await.unwrap(); + assert!(is_candidate(raft.role.as_i32())); + + raft.handle_internal_event(InternalEvent::LogPurgeCompleted(LogId { + term: 1, + index: 50, + })) + .await + .expect("LogPurgeCompleted on Candidate must return Ok (RoleViolation swallowed)"); + + assert!( + is_candidate(raft.role.as_i32()), + "Role must remain Candidate" + ); +} + // ============================================================================ // C. LEADER INITIALIZATION TESTS (Peer State Setup) // ============================================================================ diff --git a/d-engine-server/src/membership/raft_membership.rs b/d-engine-server/src/membership/raft_membership.rs index 672d2ef8..5758fb9c 100644 --- a/d-engine-server/src/membership/raft_membership.rs +++ b/d-engine-server/src/membership/raft_membership.rs @@ -759,7 +759,7 @@ impl RaftMembership where T: TypeConfig, { - /// Returns `(Self, zombie_rx)`. Caller must pass `zombie_rx` to the inbound event loop + /// Returns `(Self, zombie_rx)`. Caller must pass `zombie_rx` to the internal event loop /// so that `ZombieDetected` signals reach the leader. /// /// Call [`subscribe_membership`] on the returned `Self` to obtain a @@ -820,7 +820,7 @@ where /// Returns true if the zombie signal for `node_id` should still be acted on. /// /// Delegates to the inner health monitor. The bridge task calls this before - /// forwarding a `ZombieDetected` signal to the inbound event loop so that a + /// forwarding a `ZombieDetected` signal to the internal event loop so that a /// stale signal (peer already recovered via `record_success`) is dropped. pub(crate) fn is_zombie_valid( &self, From 9d96f30e4bb6bbd356193d9b70fdad2000abde65 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:31:03 +0800 Subject: [PATCH 4/4] test #413: cover StreamSnapshot/JoinCluster rejection and DiscoverLeader redirect --- .../src/raft_role/candidate_state_test.rs | 107 ++++++++++++++++++ .../src/raft_role/follower_state_test.rs | 49 ++++++++ .../leader_state_test/event_handling_test.rs | 53 +++++++++ .../src/raft_role/learner_state_test.rs | 48 ++++++++ 4 files changed, 257 insertions(+) diff --git a/d-engine-core/src/raft_role/candidate_state_test.rs b/d-engine-core/src/raft_role/candidate_state_test.rs index 1fac943a..04a8daa8 100644 --- a/d-engine-core/src/raft_role/candidate_state_test.rs +++ b/d-engine-core/src/raft_role/candidate_state_test.rs @@ -1344,3 +1344,110 @@ async fn test_new_leader_initializes_empty_buffers() { panic!("Expected Leader state after BecomeLeader"); } } + +// ============================================================================ +// Unexpected InboundEvent Rejection Tests +// ============================================================================ + +/// Test: Candidate rejects JoinCluster — only Leader handles cluster join requests. +/// +/// Scenario: +/// - Candidate receives JoinCluster (e.g. misdirected during election) +/// +/// Expected: +/// - sender receives Err(PermissionDenied) +/// - handle_inbound_event returns Err(RoleViolation) — non-fatal at dispatch layer +#[tokio::test] +async fn test_candidate_rejects_join_cluster() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let context = crate::test_utils::mock::mock_raft_context( + "/tmp/test_candidate_rejects_join_cluster", + graceful_rx, + None, + ); + + let mut state = crate::raft_role::candidate_state::CandidateState::< + crate::test_utils::mock::MockTypeConfig, + >::new(1, context.node_config.clone()); + + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let request = d_engine_proto::server::cluster::JoinRequest { + node_id: 5, + address: "10.0.0.5:50051".to_string(), + node_role: d_engine_proto::common::NodeRole::Learner.into(), + status: d_engine_proto::common::NodeStatus::Promotable as i32, + }; + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::JoinCluster(request, resp_tx), + &context, + internal_event_tx, + ) + .await; + + // JoinCluster on Candidate returns Err(RoleViolation) — dispatch layer swallows it + assert!( + result.is_err(), + "JoinCluster on Candidate must return RoleViolation" + ); + + let response = resp_rx.recv().await.expect("sender must reply"); + assert!(response.is_err()); + assert_eq!( + response.unwrap_err().code(), + Code::PermissionDenied, + "Candidate must reply PermissionDenied for JoinCluster" + ); +} + +/// Test: Candidate rejects StreamSnapshot — only Leader streams snapshots. +/// +/// Scenario: +/// - Candidate receives StreamSnapshot (misdirected during election) +/// +/// Expected: +/// - startup_tx receives Err(FailedPrecondition) +/// - handle_inbound_event returns Ok() (not a fatal error) +#[tokio::test] +async fn test_candidate_rejects_stream_snapshot() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let context = crate::test_utils::mock::mock_raft_context( + "/tmp/test_candidate_rejects_stream_snapshot", + graceful_rx, + None, + ); + + let mut state = crate::raft_role::candidate_state::CandidateState::< + crate::test_utils::mock::MockTypeConfig, + >::new(1, context.node_config.clone()); + + let (_ack_tx, ack_rx) = + tokio::sync::mpsc::channel::(4); + let (chunk_tx, _chunk_rx) = tokio::sync::mpsc::channel::< + std::sync::Arc, + >(4); + let (startup_tx, startup_rx) = + tokio::sync::oneshot::channel::>(); + + let (internal_event_tx, _internal_event_rx) = tokio::sync::mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), + &context, + internal_event_tx, + ) + .await; + + assert!(result.is_ok(), "StreamSnapshot rejection must not be fatal"); + + let startup_result = startup_rx.await.expect("startup_tx must be sent"); + assert!(startup_result.is_err()); + assert_eq!( + startup_result.unwrap_err().code(), + tonic::Code::FailedPrecondition, + "Candidate must reply FailedPrecondition for StreamSnapshot" + ); +} diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index 9cd0b4f3..11045d7f 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -2908,3 +2908,52 @@ async fn test_follower_cluster_conf_always_exposes_current_leader() { "follower must expose known leader ID" ); } + +// ============================================================================ +// StreamSnapshot Rejection Tests +// ============================================================================ + +/// Test: Follower rejects StreamSnapshot — only Leader streams snapshots to Learners. +/// +/// Scenario: +/// - Follower receives StreamSnapshot (misdirected from a Learner) +/// +/// Expected: +/// - startup_tx receives Err(FailedPrecondition) — tells caller this node is not the leader +/// - handle_inbound_event returns Ok() (not a fatal error) +#[tokio::test] +async fn test_follower_rejects_stream_snapshot() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + + let (_ack_tx, ack_rx) = + tokio::sync::mpsc::channel::(4); + let (chunk_tx, _chunk_rx) = tokio::sync::mpsc::channel::< + std::sync::Arc, + >(4); + let (startup_tx, startup_rx) = + tokio::sync::oneshot::channel::>(); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), + &context, + internal_event_tx, + ) + .await; + + assert!(result.is_ok(), "StreamSnapshot rejection must not be fatal"); + + let startup_result = startup_rx.await.expect("startup_tx must be sent"); + assert!(startup_result.is_err()); + assert_eq!( + startup_result.unwrap_err().code(), + tonic::Code::FailedPrecondition, + "Follower must reply FailedPrecondition — not the leader" + ); +} diff --git a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs index 57a1128d..089918c4 100644 --- a/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/event_handling_test.rs @@ -1221,3 +1221,56 @@ async fn test_cluster_conf_leader_id_transitions_after_noop_commits() { "leader must be visible after noop commits" ); } + +/// Test: Leader routes JoinCluster through handle_inbound_event to handle_join_cluster. +/// +/// Exercises the InboundEvent::JoinCluster dispatch arm in handle_inbound_event. +/// Uses a node that already exists in the cluster to trigger an early-return path +/// in handle_join_cluster without requiring full replication mock setup. +/// +/// Expected: +/// - sender receives Err(FailedPrecondition) — NodeAlreadyExists +/// - handle_inbound_event propagates the non-fatal error (swallowed at dispatch layer) +#[tokio::test] +async fn test_leader_join_cluster_dispatched_via_handle_inbound_event() { + use crate::maybe_clone_oneshot::MaybeCloneOneshot; + use d_engine_proto::server::cluster::JoinRequest; + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut context = + mock_raft_context("/tmp/test_leader_join_cluster_dispatch", graceful_rx, None); + + let mut membership = create_mock_membership(); + // Node 5 already in cluster → handle_join_cluster returns early via send_join_error + membership.expect_contains_node().returning(|_| true); + context.membership = Arc::new(membership); + + let mut state = LeaderState::::new(1, context.node_config()); + + let (resp_tx, mut resp_rx) = >::new(); + let request = JoinRequest { + node_id: 5, + address: "10.0.0.5:50051".to_string(), + node_role: d_engine_proto::common::NodeRole::Learner.into(), + status: d_engine_proto::common::NodeStatus::Promotable as i32, + }; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::JoinCluster(request, resp_tx), + &context, + internal_event_tx, + ) + .await; + + // handle_join_cluster returns Err(NodeAlreadyExists) — non-fatal, swallowed by dispatch + assert!( + result.is_err(), + "NodeAlreadyExists must propagate out of handle_inbound_event" + ); + + let response = resp_rx.recv().await.expect("sender must reply"); + assert!(response.is_err(), "client must receive an error response"); + assert_eq!(response.unwrap_err().code(), Code::FailedPrecondition); +} diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index 17c20fd4..4ca886ce 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -1955,3 +1955,51 @@ async fn test_learner_install_snapshot_reports_failure_when_apply_fails_after_tr "Learner must NOT report success when apply failed after transfer (got success:true — #308 bug)" ); } + +// ============================================================================ +// StreamSnapshot Rejection Tests +// ============================================================================ + +/// Test: Learner rejects StreamSnapshot — Learner is the requester, not the server. +/// +/// Scenario: +/// - Learner receives StreamSnapshot (invalid: Learner pulls from Leader, not the reverse) +/// +/// Expected: +/// - startup_tx receives Err(FailedPrecondition) +/// - handle_inbound_event returns Ok() (not a fatal error) +#[tokio::test] +async fn test_learner_rejects_stream_snapshot() { + let (_graceful_tx, graceful_rx) = tokio::sync::watch::channel(()); + let (context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let mut state = LearnerState::::new(1, context.node_config.clone()); + + let (_ack_tx, ack_rx) = + tokio::sync::mpsc::channel::(4); + let (chunk_tx, _chunk_rx) = tokio::sync::mpsc::channel::< + std::sync::Arc, + >(4); + let (startup_tx, startup_rx) = + tokio::sync::oneshot::channel::>(); + + let (internal_event_tx, _) = tokio::sync::mpsc::unbounded_channel(); + + let result = state + .handle_inbound_event( + InboundEvent::StreamSnapshot(ack_rx, chunk_tx, startup_tx), + &context, + internal_event_tx, + ) + .await; + + assert!(result.is_ok(), "StreamSnapshot rejection must not be fatal"); + + let startup_result = startup_rx.await.expect("startup_tx must be sent"); + assert!(startup_result.is_err()); + assert_eq!( + startup_result.unwrap_err().code(), + tonic::Code::FailedPrecondition, + "Learner must reply FailedPrecondition for StreamSnapshot" + ); +}