//! Layer 2 — behavioral tests (in-process). Goals 1, 2, 3, 6, 7 of //! BEHAVIORAL_TEST_SPEC.md. //! //! These observe ONLY the boundary the spec draws: each node's membership view //! (`members()`) and the `MembershipChanged` notification stream. They never //! inspect probe phases, piggyback bytes, queue contents, timers, or action //! enums in an assertion — so they survive any rewrite that keeps the boundary. //! The harness DOES route `Ping`/`Ack`/`PingReq`/`IndirectAck`/`JoinResponse` //! between nodes, but that is the simulated transport, not a test assertion. //! //! Eventuality is pinned by driving the cluster to quiescence — repeated gossip //! rounds until several consecutive rounds produce no new `MembershipChanged` — //! and then asserting on the settled view. No fixed round count is tuned to pass. use std::collections::BTreeMap; use std::time::{Duration, Instant}; use distribution::swim::dissemination::{membership_update, DisseminationQueue}; use distribution::swim::member_list::MemberList; use distribution::swim::node::{NodeAction, SwimNode}; use distribution::swim::probe::{ProbeMode, SwimConfig}; use distribution::types::{MemberState, NodeId}; const TICK: Duration = Duration::from_millis(10); fn t(n: u64) -> Duration { TICK * n as u32 } fn id(byte: u8) -> NodeId { NodeId([byte; 32]) } /// A brisk Periodic config so detection completes in a manageable number of /// rounds: detection ≈ 2·probe_timeout (direct+indirect) + suspicion_timeout. fn behavioral_config(dead_reprobe: Duration) -> SwimConfig { SwimConfig { probe_interval: t(1), probe_timeout: t(2), indirect_probes: 2, suspicion_timeout: t(3), dead_reprobe_interval: dead_reprobe, probe_mode: ProbeMode::Periodic, lifeguard: None, } } /// A single update about `target`, encoded as a wire piggyback frame. fn piggyback_about(target: NodeId, state: MemberState, incarnation: u64) -> Vec { let mut q = DisseminationQueue::new(3); q.enqueue(membership_update(target, state, incarnation), 5); q.pack_piggyback(10) } type View = BTreeMap<[u8; 32], (MemberState, u64)>; /// Fold a `MembershipChanged` stream into a view using the §7 merge rule — the /// dominating update per node. This is the Goal-7 reconstruction. fn fold_stream(stream: &[(NodeId, MemberState, u64)]) -> View { // self id is a sentinel that never appears in any stream, so every entry is // stored (self is never stored, §7 inv. 4). let mut ml = MemberList::new(id(250)); for &(nid, st, inc) in stream { ml.apply(nid, st, inc); } ml.all_members().iter().map(|e| (e.node_id.0, (e.state, e.incarnation))).collect() } /// In-process cluster of `SwimNode`s with simulated, correct-by-construction /// transport. Records each node's emitted `MembershipChanged` stream. struct Cluster { ids: Vec, nodes: Vec, clock: Instant, notifications: Vec>, } impl Cluster { /// `n` nodes; nodes 1..n join via node 0 (the seed). fn new(n: usize, config: SwimConfig) -> Self { let now = Instant::now(); let ids: Vec = (0..n).map(|i| id(i as u8)).collect(); let nodes = ids.iter().map(|&nid| SwimNode::new(nid, config.clone(), now)).collect(); let mut c = Cluster { ids, nodes, clock: now, notifications: vec![Vec::new(); n] }; for i in 1..n { let acts = c.nodes[0].handle_join_request(c.ids[i]); c.record(0, &acts); c.route(0, acts, &[]); } c } fn index_of(&self, nid: NodeId) -> Option { self.ids.iter().position(|x| *x == nid) } fn record(&mut self, origin: usize, acts: &[NodeAction]) { for a in acts { if let NodeAction::MembershipChanged { node_id, state, incarnation } = a { self.notifications[origin].push((*node_id, *state, *incarnation)); } } } /// Deliver every network action `origin` produced to its target, recursively /// routing each response. `excluded` nodes neither send nor receive (genuine /// silence). MembershipChanged is captured by `record`, never delivered. fn route(&mut self, origin: usize, acts: Vec, excluded: &[usize]) { if excluded.contains(&origin) { return; } for a in acts { match a { NodeAction::SendPing { to, sequence, piggyback } => { if let Some(t) = self.index_of(to) { if t != origin && !excluded.contains(&t) { let from = self.ids[origin]; let resp = self.nodes[t].handle_ping(from, sequence, &piggyback); self.record(t, &resp); self.route(t, resp, excluded); } } } NodeAction::SendAck { to, sequence, piggyback } => { if let Some(t) = self.index_of(to) { if t != origin && !excluded.contains(&t) { let from = self.ids[origin]; let resp = self.nodes[t].handle_ack(from, sequence, &piggyback); self.record(t, &resp); self.route(t, resp, excluded); } } } NodeAction::SendPingReq { relay, target, sequence, piggyback } => { if let Some(t) = self.index_of(relay) { if t != origin && !excluded.contains(&t) { let from = self.ids[origin]; let resp = self.nodes[t].handle_ping_req(from, target, sequence, &piggyback); self.record(t, &resp); self.route(t, resp, excluded); } } } NodeAction::ForwardAck { to, target, sequence, piggyback } => { if let Some(t) = self.index_of(to) { if t != origin && !excluded.contains(&t) { let resp = self.nodes[t].handle_indirect_ack(target, sequence, &piggyback); self.record(t, &resp); self.route(t, resp, excluded); } } } NodeAction::SendJoinResponse { to, members } => { if let Some(t) = self.index_of(to) { if t != origin && !excluded.contains(&t) { let resp = self.nodes[t].handle_join_response(members); self.record(t, &resp); self.route(t, resp, excluded); } } } NodeAction::MembershipChanged { .. } => {} } } } fn gossip_round(&mut self, excluded: &[usize]) { self.clock += TICK; for i in 0..self.nodes.len() { if excluded.contains(&i) { continue; } let now = self.clock; let acts = self.nodes[i].tick(now); self.record(i, &acts); self.route(i, acts, excluded); } } /// Converge-or-timeout: drive rounds until the eventual `cond` holds, or /// `cap` rounds elapse. Returns whether `cond` was reached. This pins /// eventuality without a fixed count — and, unlike "no new notifications", /// it does not mistake an in-flight probe timeout for a settled cluster. fn run_until bool>(&mut self, excluded: &[usize], cap: usize, cond: F) -> bool { if cond(self) { return true; } for _ in 0..cap { self.gossip_round(excluded); if cond(self) { return true; } } false } fn state_of(&self, observer: usize, subject: NodeId) -> Option { self.nodes[observer].members().get(&subject).map(|e| e.state) } /// Every node sees every other node as Alive (shared converged view). fn all_converged_alive(&self) -> bool { let n = self.nodes.len(); (0..n).all(|o| { (0..n).all(|s| o == s || self.state_of(o, id(s as u8)) == Some(MemberState::Alive)) }) } /// Every non-excluded node sees `subject` in `state`. fn survivors_see(&self, excluded: &[usize], subject: NodeId, state: MemberState) -> bool { (0..self.nodes.len()).all(|o| excluded.contains(&o) || self.state_of(o, subject) == Some(state)) } fn view(&self, observer: usize) -> View { self.nodes[observer] .members() .all_members() .iter() .map(|e| (e.node_id.0, (e.state, e.incarnation))) .collect() } } // ─── Goal 1 — convergence ──────────────────────────────────────────────────── #[test] fn goal1_nodes_join_and_reach_a_shared_alive_view() { let mut c = Cluster::new(4, behavioral_config(t(0))); assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "cluster did not converge to a shared Alive view"); for observer in 0..4 { assert_eq!(c.nodes[observer].members().alive_count(), 3, "node {observer} must see all 3 peers alive"); } } // ─── Goal 2 — real detection ───────────────────────────────────────────────── #[test] fn goal2_a_truly_silent_node_is_detected_dead_by_survivors() { let mut c = Cluster::new(4, behavioral_config(t(0))); assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "precondition: cluster must converge"); // Genuinely silence node 3 — it neither ticks nor sends nor receives, so a // survivor's probe TRULY times out (not an injected death). Poll until the // survivors converge on it being Dead, or time out. let dead = 3usize; let detected = c.run_until(&[dead], 1000, |c| c.survivors_see(&[dead], id(3), MemberState::Dead)); assert!(detected, "survivors must converge on the silenced node being Dead within the detection window"); } // ─── Goal 3 — death is provisional ─────────────────────────────────────────── #[test] fn goal3_a_silenced_node_resurrects_when_it_answers_again() { // dead_reprobe enabled so the partition-heal detector (§9.8) re-probes the // Dead node and lets it refute back to Alive. let mut c = Cluster::new(4, behavioral_config(t(2))); assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "precondition: cluster must converge"); let isolated = 3usize; let died = c.run_until(&[isolated], 1000, |c| c.survivors_see(&[isolated], id(3), MemberState::Dead)); assert!(died, "the silenced node must first be detected Dead by the survivors"); // Restore the node: it answers probes again, so the reprobe revives it. let revived = c.run_until(&[], 1000, |c| c.survivors_see(&[isolated], id(3), MemberState::Alive)); assert!(revived, "the restored node must resurrect to Alive (death is provisional)"); // A survivor's notification stream witnessed the full provisional arc. let s = if isolated == 0 { 1 } else { 0 }; let stream = &c.notifications[s]; let dead_at = stream.iter().position(|(n, st, _)| *n == id(isolated as u8) && *st == MemberState::Dead); let alive_after = stream.iter().rposition(|(n, st, _)| *n == id(isolated as u8) && *st == MemberState::Alive); assert!( matches!((dead_at, alive_after), (Some(d), Some(a)) if a > d), "a survivor must witness node {isolated} go Dead then back Alive" ); } // ─── Goal 6 — dissemination reaches everyone ───────────────────────────────── #[test] fn goal6_a_single_change_known_to_one_node_infects_every_node() { let mut c = Cluster::new(4, behavioral_config(t(0))); assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "precondition: cluster must converge"); // A change known to ONLY node 0: a phantom peer reported Dead in a single // gossip exchange. No other node has ever heard of this peer, and no node // probes a Dead member — so the others can learn it ONLY by multi-hop // piggyback infection, never by direct observation. let phantom = id(9); let from = c.ids[1]; let pb = piggyback_about(phantom, MemberState::Dead, 5); let _ = c.nodes[0].handle_ping(from, 1, &pb); let infected = c.run_until(&[], 1000, |c| c.survivors_see(&[], phantom, MemberState::Dead)); assert!(infected, "the single change must infect every node — dissemination reaches everyone, not just probe partners"); } // ─── Goal 7 — notification contract ────────────────────────────────────────── #[test] fn goal7_membership_changed_stream_reconstructs_the_view() { // §6.3 / Move 1: `MembershipChanged` is the sole observable and the stream // alone must reconstruct the settled membership view. (May be red until the // §10.2 learn-sender notification lands — a peer learned via an inbound Ping // mutates the view today without emitting MembershipChanged.) let mut c = Cluster::new(4, behavioral_config(t(0))); assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "cluster did not converge"); for observer in 0..4 { assert_eq!( fold_stream(&c.notifications[observer]), c.view(observer), "node {observer}: folding the MembershipChanged stream must reconstruct the view" ); } } #[test] fn goal7_stream_reconstructs_view_for_a_peer_learned_via_ping() { // §6.3 / §10.2: even a peer first learned by receiving its Ping must appear in // the notification stream. EXPECTED RED today — learning a sender mutates the // view without a MembershipChanged; the actor must route it through the stream. let now = Instant::now(); let mut a = SwimNode::new(id(0), behavioral_config(t(0)), now); let mut stream: Vec<(NodeId, MemberState, u64)> = Vec::new(); for act in a.handle_ping(id(5), 1, &[]) { if let NodeAction::MembershipChanged { node_id, state, incarnation } = act { stream.push((node_id, state, incarnation)); } } let view: View = a .members() .all_members() .iter() .map(|e| (e.node_id.0, (e.state, e.incarnation))) .collect(); assert_eq!( fold_stream(&stream), view, "the stream must reconstruct the view even for a peer learned via its Ping (§10.2)" ); } #[test] fn goal7_no_duplicate_or_coalesced_notifications() { // §6.3: one notification per real change — no dupes, no coalescing. Each // notification for a given peer must strictly advance (dominate) the previous // one for that peer; an identical repeat would be a spurious duplicate. let mut c = Cluster::new(4, behavioral_config(t(0))); assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "cluster did not converge"); for observer in 0..4 { let mut last: BTreeMap<[u8; 32], (MemberState, u64)> = BTreeMap::new(); for &(nid, st, inc) in &c.notifications[observer] { if let Some(&(pst, pinc)) = last.get(&nid.0) { let advances = inc > pinc || (inc == pinc && st.priority() > pst.priority()); assert!( advances, "node {observer}: a notification ({st:?},{inc}) did not advance past ({pst:?},{pinc}) — duplicate/coalesced" ); } last.insert(nid.0, (st, inc)); } } }