zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: guard SWIM suspicion timeout against refuted nodes

Browse source 
A stale suspicion timer could declare a node dead even after it had been
refuted back to Alive via a higher incarnation number. check_suspicion_timeouts
now verifies the node is still Suspect before calling declare_dead, and
cancels the probe phase if the dead node was the current target.

Also fix the 3-node gossip loop in node_death_tombstones_entries to
deliver responses with the correct sender_id — previously all responses
were attributed to a single hardcoded sender, causing false suspicions
that triggered the protocol bug.

Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
This commit is contained in:
Claude 2026-02-15 06:35:01 +00:00
parent 784a7d42da
commit 205dc2352f
2 changed files with 45 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
crates/distribution/src/swim/probe.rs
View file

@ -5,7 +5,7 @@
use std::collections::VecDeque;
use crate::types::NodeId;
use crate::types::{MemberState, NodeId};
use super::member_list::MemberList;
@ -342,8 +342,26 @@ impl SwimProbe {
.collect();
for node_id in expired {
if members.declare_dead(node_id) {
// Only declare dead if still suspect. A refutation (Alive with
// higher incarnation) clears the suspect state in MemberList;
// honour that by dropping the stale timer instead of killing the node.
let still_suspect = members
.get(&node_id)
.is_some_and(|e| e.state == MemberState::Suspect);
if still_suspect && members.declare_dead(node_id) {
actions.push(SwimAction::DeclareDead(node_id));
// If we're currently probing the dead node, cancel immediately
// so we can probe live members on this same tick.
match &self.phase {
ProbePhase::WaitingDirectAck { target, .. }
| ProbePhase::WaitingIndirectAck { target, .. }
if *target == node_id =>
{
self.phase = ProbePhase::Idle;
}
_ => {}
}
}
self.cancel_suspicion_timer(node_id);
}

37
crates/distribution/tests/registry.rs
View file

@ -309,24 +309,37 @@ fn node_death_tombstones_entries() {
b.register_name("b-service".into(), actor);
// Propagate B's registration to A and C via mesh gossip.
// Deliver to each target separately so responses carry the correct sender_id.
for _ in 0..5 {
let actions = b.tick();
let mut nodes = vec![(a_id, &mut a), (c_id, &mut c)];
let responses = deliver_actions(&actions, b_id, &mut nodes);
let mut nodes = vec![(b_id, &mut b)];
let _ = deliver_actions(&responses, a_id, &mut nodes);
let mut t = vec![(a_id, &mut a)];
let resp_a = deliver_actions(&actions, b_id, &mut t);
let mut t = vec![(c_id, &mut c)];
let resp_c = deliver_actions(&actions, b_id, &mut t);
let mut t = vec![(b_id, &mut b)];
let _ = deliver_actions(&resp_a, a_id, &mut t);
let mut t = vec![(b_id, &mut b)];
let _ = deliver_actions(&resp_c, c_id, &mut t);
let actions = a.tick();
let mut nodes = vec![(b_id, &mut b), (c_id, &mut c)];
let responses = deliver_actions(&actions, a_id, &mut nodes);
let mut nodes = vec![(a_id, &mut a)];
let _ = deliver_actions(&responses, b_id, &mut nodes);
let mut t = vec![(b_id, &mut b)];
let resp_b = deliver_actions(&actions, a_id, &mut t);
let mut t = vec![(c_id, &mut c)];
let resp_c = deliver_actions(&actions, a_id, &mut t);
let mut t = vec![(a_id, &mut a)];
let _ = deliver_actions(&resp_b, b_id, &mut t);
let mut t = vec![(a_id, &mut a)];
let _ = deliver_actions(&resp_c, c_id, &mut t);
let actions = c.tick();
let mut nodes = vec![(a_id, &mut a), (b_id, &mut b)];
let responses = deliver_actions(&actions, c_id, &mut nodes);
let mut nodes = vec![(c_id, &mut c)];
let _ = deliver_actions(&responses, a_id, &mut nodes);
let mut t = vec![(a_id, &mut a)];
let resp_a = deliver_actions(&actions, c_id, &mut t);
let mut t = vec![(b_id, &mut b)];
let resp_b = deliver_actions(&actions, c_id, &mut t);
let mut t = vec![(c_id, &mut c)];
let _ = deliver_actions(&resp_a, a_id, &mut t);
let mut t = vec![(c_id, &mut c)];
let _ = deliver_actions(&resp_b, b_id, &mut t);
}
assert_eq!(a.resolve_name("b-service"), Some((actor, b_id)));