From 205dc2352fff0ed1cd5477848384367fb9f2f264 Mon Sep 17 00:00:00 2001
From: Claude <claude@swactor.dev>
Date: Sun, 15 Feb 2026 06:35:01 +0000
Subject: [PATCH] fix: guard SWIM suspicion timeout against refuted nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A stale suspicion timer could declare a node dead even after it had been
refuted back to Alive via a higher incarnation number. check_suspicion_timeouts
now verifies the node is still Suspect before calling declare_dead, and
cancels the probe phase if the dead node was the current target.

Also fix the 3-node gossip loop in node_death_tombstones_entries to
deliver responses with the correct sender_id — previously all responses
were attributed to a single hardcoded sender, causing false suspicions
that triggered the protocol bug.

Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
---
 crates/distribution/src/swim/probe.rs | 22 ++++++++++++++--
 crates/distribution/tests/registry.rs | 37 ++++++++++++++++++---------
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/crates/distribution/src/swim/probe.rs b/crates/distribution/src/swim/probe.rs
index 31750da..a85deea 100644
--- a/crates/distribution/src/swim/probe.rs
+++ b/crates/distribution/src/swim/probe.rs
@@ -5,7 +5,7 @@
 
 use std::collections::VecDeque;
 
-use crate::types::NodeId;
+use crate::types::{MemberState, NodeId};
 
 use super::member_list::MemberList;
 
@@ -342,8 +342,26 @@ impl SwimProbe {
             .collect();
 
         for node_id in expired {
-            if members.declare_dead(node_id) {
+            // Only declare dead if still suspect. A refutation (Alive with
+            // higher incarnation) clears the suspect state in MemberList;
+            // honour that by dropping the stale timer instead of killing the node.
+            let still_suspect = members
+                .get(&node_id)
+                .is_some_and(|e| e.state == MemberState::Suspect);
+
+            if still_suspect && members.declare_dead(node_id) {
                 actions.push(SwimAction::DeclareDead(node_id));
+                // If we're currently probing the dead node, cancel immediately
+                // so we can probe live members on this same tick.
+                match &self.phase {
+                    ProbePhase::WaitingDirectAck { target, .. }
+                    | ProbePhase::WaitingIndirectAck { target, .. }
+                        if *target == node_id =>
+                    {
+                        self.phase = ProbePhase::Idle;
+                    }
+                    _ => {}
+                }
             }
             self.cancel_suspicion_timer(node_id);
         }
diff --git a/crates/distribution/tests/registry.rs b/crates/distribution/tests/registry.rs
index 19d6760..1b92681 100644
--- a/crates/distribution/tests/registry.rs
+++ b/crates/distribution/tests/registry.rs
@@ -309,24 +309,37 @@ fn node_death_tombstones_entries() {
     b.register_name("b-service".into(), actor);
 
     // Propagate B's registration to A and C via mesh gossip.
+    // Deliver to each target separately so responses carry the correct sender_id.
     for _ in 0..5 {
         let actions = b.tick();
-        let mut nodes = vec![(a_id, &mut a), (c_id, &mut c)];
-        let responses = deliver_actions(&actions, b_id, &mut nodes);
-        let mut nodes = vec![(b_id, &mut b)];
-        let _ = deliver_actions(&responses, a_id, &mut nodes);
+        let mut t = vec![(a_id, &mut a)];
+        let resp_a = deliver_actions(&actions, b_id, &mut t);
+        let mut t = vec![(c_id, &mut c)];
+        let resp_c = deliver_actions(&actions, b_id, &mut t);
+        let mut t = vec![(b_id, &mut b)];
+        let _ = deliver_actions(&resp_a, a_id, &mut t);
+        let mut t = vec![(b_id, &mut b)];
+        let _ = deliver_actions(&resp_c, c_id, &mut t);
 
         let actions = a.tick();
-        let mut nodes = vec![(b_id, &mut b), (c_id, &mut c)];
-        let responses = deliver_actions(&actions, a_id, &mut nodes);
-        let mut nodes = vec![(a_id, &mut a)];
-        let _ = deliver_actions(&responses, b_id, &mut nodes);
+        let mut t = vec![(b_id, &mut b)];
+        let resp_b = deliver_actions(&actions, a_id, &mut t);
+        let mut t = vec![(c_id, &mut c)];
+        let resp_c = deliver_actions(&actions, a_id, &mut t);
+        let mut t = vec![(a_id, &mut a)];
+        let _ = deliver_actions(&resp_b, b_id, &mut t);
+        let mut t = vec![(a_id, &mut a)];
+        let _ = deliver_actions(&resp_c, c_id, &mut t);
 
         let actions = c.tick();
-        let mut nodes = vec![(a_id, &mut a), (b_id, &mut b)];
-        let responses = deliver_actions(&actions, c_id, &mut nodes);
-        let mut nodes = vec![(c_id, &mut c)];
-        let _ = deliver_actions(&responses, a_id, &mut nodes);
+        let mut t = vec![(a_id, &mut a)];
+        let resp_a = deliver_actions(&actions, c_id, &mut t);
+        let mut t = vec![(b_id, &mut b)];
+        let resp_b = deliver_actions(&actions, c_id, &mut t);
+        let mut t = vec![(c_id, &mut c)];
+        let _ = deliver_actions(&resp_a, a_id, &mut t);
+        let mut t = vec![(c_id, &mut c)];
+        let _ = deliver_actions(&resp_b, b_id, &mut t);
     }
 
     assert_eq!(a.resolve_name("b-service"), Some((actor, b_id)));