swactor/crates/distribution/tests/swim_behavioral.rs

//! Layer 2 — behavioral tests (in-process). Goals 1, 2, 3, 6, 7 of
//! BEHAVIORAL_TEST_SPEC.md.
//!
//! These observe ONLY the boundary the spec draws: each node's membership view
//! (`members()`) and the `MembershipChanged` notification stream. They never
//! inspect probe phases, piggyback bytes, queue contents, timers, or action
//! enums in an assertion — so they survive any rewrite that keeps the boundary.
//! The harness DOES route `Ping`/`Ack`/`PingReq`/`IndirectAck`/`JoinResponse`
//! between nodes, but that is the simulated transport, not a test assertion.
//!
//! Eventuality is pinned by driving the cluster to quiescence — repeated gossip
//! rounds until several consecutive rounds produce no new `MembershipChanged` —
//! and then asserting on the settled view. No fixed round count is tuned to pass.

use std::collections::BTreeMap;
use std::time::{Duration, Instant};

use distribution::swim::dissemination::{membership_update, DisseminationQueue};
use distribution::swim::member_list::MemberList;
use distribution::swim::node::{NodeAction, SwimNode};
use distribution::swim::probe::{ProbeMode, SwimConfig};
use distribution::types::{MemberState, NodeId};

const TICK: Duration = Duration::from_millis(10);

fn t(n: u64) -> Duration {
    TICK * n as u32
}

fn id(byte: u8) -> NodeId {
    NodeId([byte; 32])
}

/// A brisk Periodic config so detection completes in a manageable number of
/// rounds: detection ≈ 2·probe_timeout (direct+indirect) + suspicion_timeout.
fn behavioral_config(dead_reprobe: Duration) -> SwimConfig {
    SwimConfig {
        probe_interval: t(1),
        probe_timeout: t(2),
        indirect_probes: 2,
        suspicion_timeout: t(3),
        dead_reprobe_interval: dead_reprobe,
        probe_mode: ProbeMode::Periodic,
        lifeguard: None,
    }
}

/// A single update about `target`, encoded as a wire piggyback frame.
fn piggyback_about(target: NodeId, state: MemberState, incarnation: u64) -> Vec<u8> {
    let mut q = DisseminationQueue::new(3);
    q.enqueue(membership_update(target, state, incarnation), 5);
    q.pack_piggyback(10)
}

type View = BTreeMap<[u8; 32], (MemberState, u64)>;

/// Fold a `MembershipChanged` stream into a view using the §7 merge rule — the
/// dominating update per node. This is the Goal-7 reconstruction.
fn fold_stream(stream: &[(NodeId, MemberState, u64)]) -> View {
    // self id is a sentinel that never appears in any stream, so every entry is
    // stored (self is never stored, §7 inv. 4).
    let mut ml = MemberList::new(id(250));
    for &(nid, st, inc) in stream {
        ml.apply(nid, st, inc);
    }
    ml.all_members().iter().map(|e| (e.node_id.0, (e.state, e.incarnation))).collect()
}

/// In-process cluster of `SwimNode`s with simulated, correct-by-construction
/// transport. Records each node's emitted `MembershipChanged` stream.
struct Cluster {
    ids: Vec<NodeId>,
    nodes: Vec<SwimNode>,
    clock: Instant,
    notifications: Vec<Vec<(NodeId, MemberState, u64)>>,
}

impl Cluster {
    /// `n` nodes; nodes 1..n join via node 0 (the seed).
    fn new(n: usize, config: SwimConfig) -> Self {
        let now = Instant::now();
        let ids: Vec<NodeId> = (0..n).map(|i| id(i as u8)).collect();
        let nodes = ids.iter().map(|&nid| SwimNode::new(nid, config.clone(), now)).collect();
        let mut c = Cluster { ids, nodes, clock: now, notifications: vec![Vec::new(); n] };
        for i in 1..n {
            let acts = c.nodes[0].handle_join_request(c.ids[i]);
            c.record(0, &acts);
            c.route(0, acts, &[]);
        }
        c
    }

    fn index_of(&self, nid: NodeId) -> Option<usize> {
        self.ids.iter().position(|x| *x == nid)
    }

    fn record(&mut self, origin: usize, acts: &[NodeAction]) {
        for a in acts {
            if let NodeAction::MembershipChanged { node_id, state, incarnation } = a {
                self.notifications[origin].push((*node_id, *state, *incarnation));
            }
        }
    }

    /// Deliver every network action `origin` produced to its target, recursively
    /// routing each response. `excluded` nodes neither send nor receive (genuine
    /// silence). MembershipChanged is captured by `record`, never delivered.
    fn route(&mut self, origin: usize, acts: Vec<NodeAction>, excluded: &[usize]) {
        if excluded.contains(&origin) {
            return;
        }
        for a in acts {
            match a {
                NodeAction::SendPing { to, sequence, piggyback } => {
                    if let Some(t) = self.index_of(to) {
                        if t != origin && !excluded.contains(&t) {
                            let from = self.ids[origin];
                            let resp = self.nodes[t].handle_ping(from, sequence, &piggyback);
                            self.record(t, &resp);
                            self.route(t, resp, excluded);
                        }
                    }
                }
                NodeAction::SendAck { to, sequence, piggyback } => {
                    if let Some(t) = self.index_of(to) {
                        if t != origin && !excluded.contains(&t) {
                            let from = self.ids[origin];
                            let resp = self.nodes[t].handle_ack(from, sequence, &piggyback);
                            self.record(t, &resp);
                            self.route(t, resp, excluded);
                        }
                    }
                }
                NodeAction::SendPingReq { relay, target, sequence, piggyback } => {
                    if let Some(t) = self.index_of(relay) {
                        if t != origin && !excluded.contains(&t) {
                            let from = self.ids[origin];
                            let resp = self.nodes[t].handle_ping_req(from, target, sequence, &piggyback);
                            self.record(t, &resp);
                            self.route(t, resp, excluded);
                        }
                    }
                }
                NodeAction::ForwardAck { to, target, sequence, piggyback } => {
                    if let Some(t) = self.index_of(to) {
                        if t != origin && !excluded.contains(&t) {
                            let resp = self.nodes[t].handle_indirect_ack(target, sequence, &piggyback);
                            self.record(t, &resp);
                            self.route(t, resp, excluded);
                        }
                    }
                }
                NodeAction::SendJoinResponse { to, members } => {
                    if let Some(t) = self.index_of(to) {
                        if t != origin && !excluded.contains(&t) {
                            let resp = self.nodes[t].handle_join_response(members);
                            self.record(t, &resp);
                            self.route(t, resp, excluded);
                        }
                    }
                }
                NodeAction::MembershipChanged { .. } => {}
            }
        }
    }

    fn gossip_round(&mut self, excluded: &[usize]) {
        self.clock += TICK;
        for i in 0..self.nodes.len() {
            if excluded.contains(&i) {
                continue;
            }
            let now = self.clock;
            let acts = self.nodes[i].tick(now);
            self.record(i, &acts);
            self.route(i, acts, excluded);
        }
    }

    /// Converge-or-timeout: drive rounds until the eventual `cond` holds, or
    /// `cap` rounds elapse. Returns whether `cond` was reached. This pins
    /// eventuality without a fixed count — and, unlike "no new notifications",
    /// it does not mistake an in-flight probe timeout for a settled cluster.
    fn run_until<F: Fn(&Cluster) -> bool>(&mut self, excluded: &[usize], cap: usize, cond: F) -> bool {
        if cond(self) {
            return true;
        }
        for _ in 0..cap {
            self.gossip_round(excluded);
            if cond(self) {
                return true;
            }
        }
        false
    }

    fn state_of(&self, observer: usize, subject: NodeId) -> Option<MemberState> {
        self.nodes[observer].members().get(&subject).map(|e| e.state)
    }

    /// Every node sees every other node as Alive (shared converged view).
    fn all_converged_alive(&self) -> bool {
        let n = self.nodes.len();
        (0..n).all(|o| {
            (0..n).all(|s| o == s || self.state_of(o, id(s as u8)) == Some(MemberState::Alive))
        })
    }

    /// Every non-excluded node sees `subject` in `state`.
    fn survivors_see(&self, excluded: &[usize], subject: NodeId, state: MemberState) -> bool {
        (0..self.nodes.len()).all(|o| excluded.contains(&o) || self.state_of(o, subject) == Some(state))
    }

    fn view(&self, observer: usize) -> View {
        self.nodes[observer]
            .members()
            .all_members()
            .iter()
            .map(|e| (e.node_id.0, (e.state, e.incarnation)))
            .collect()
    }
}

// ─── Goal 1 — convergence ────────────────────────────────────────────────────

#[test]
fn goal1_nodes_join_and_reach_a_shared_alive_view() {
    let mut c = Cluster::new(4, behavioral_config(t(0)));
    assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "cluster did not converge to a shared Alive view");
    for observer in 0..4 {
        assert_eq!(c.nodes[observer].members().alive_count(), 3, "node {observer} must see all 3 peers alive");
    }
}

// ─── Goal 2 — real detection ─────────────────────────────────────────────────

#[test]
fn goal2_a_truly_silent_node_is_detected_dead_by_survivors() {
    let mut c = Cluster::new(4, behavioral_config(t(0)));
    assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "precondition: cluster must converge");

    // Genuinely silence node 3 — it neither ticks nor sends nor receives, so a
    // survivor's probe TRULY times out (not an injected death). Poll until the
    // survivors converge on it being Dead, or time out.
    let dead = 3usize;
    let detected = c.run_until(&[dead], 1000, |c| c.survivors_see(&[dead], id(3), MemberState::Dead));
    assert!(detected, "survivors must converge on the silenced node being Dead within the detection window");
}

// ─── Goal 3 — death is provisional ───────────────────────────────────────────

#[test]
fn goal3_a_silenced_node_resurrects_when_it_answers_again() {
    // dead_reprobe enabled so the partition-heal detector (§9.8) re-probes the
    // Dead node and lets it refute back to Alive.
    let mut c = Cluster::new(4, behavioral_config(t(2)));
    assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "precondition: cluster must converge");

    let isolated = 3usize;
    let died = c.run_until(&[isolated], 1000, |c| c.survivors_see(&[isolated], id(3), MemberState::Dead));
    assert!(died, "the silenced node must first be detected Dead by the survivors");

    // Restore the node: it answers probes again, so the reprobe revives it.
    let revived = c.run_until(&[], 1000, |c| c.survivors_see(&[isolated], id(3), MemberState::Alive));
    assert!(revived, "the restored node must resurrect to Alive (death is provisional)");

    // A survivor's notification stream witnessed the full provisional arc.
    let s = if isolated == 0 { 1 } else { 0 };
    let stream = &c.notifications[s];
    let dead_at = stream.iter().position(|(n, st, _)| *n == id(isolated as u8) && *st == MemberState::Dead);
    let alive_after = stream.iter().rposition(|(n, st, _)| *n == id(isolated as u8) && *st == MemberState::Alive);
    assert!(
        matches!((dead_at, alive_after), (Some(d), Some(a)) if a > d),
        "a survivor must witness node {isolated} go Dead then back Alive"
    );
}

// ─── Goal 6 — dissemination reaches everyone ─────────────────────────────────

#[test]
fn goal6_a_single_change_known_to_one_node_infects_every_node() {
    let mut c = Cluster::new(4, behavioral_config(t(0)));
    assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "precondition: cluster must converge");

    // A change known to ONLY node 0: a phantom peer reported Dead in a single
    // gossip exchange. No other node has ever heard of this peer, and no node
    // probes a Dead member — so the others can learn it ONLY by multi-hop
    // piggyback infection, never by direct observation.
    let phantom = id(9);
    let from = c.ids[1];
    let pb = piggyback_about(phantom, MemberState::Dead, 5);
    let _ = c.nodes[0].handle_ping(from, 1, &pb);

    let infected = c.run_until(&[], 1000, |c| c.survivors_see(&[], phantom, MemberState::Dead));
    assert!(infected, "the single change must infect every node — dissemination reaches everyone, not just probe partners");
}

// ─── Goal 7 — notification contract ──────────────────────────────────────────

#[test]
fn goal7_membership_changed_stream_reconstructs_the_view() {
    // §6.3 / Move 1: `MembershipChanged` is the sole observable and the stream
    // alone must reconstruct the settled membership view. (May be red until the
    // §10.2 learn-sender notification lands — a peer learned via an inbound Ping
    // mutates the view today without emitting MembershipChanged.)
    let mut c = Cluster::new(4, behavioral_config(t(0)));
    assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "cluster did not converge");
    for observer in 0..4 {
        assert_eq!(
            fold_stream(&c.notifications[observer]),
            c.view(observer),
            "node {observer}: folding the MembershipChanged stream must reconstruct the view"
        );
    }
}

#[test]
fn goal7_stream_reconstructs_view_for_a_peer_learned_via_ping() {
    // §6.3 / §10.2: even a peer first learned by receiving its Ping must appear in
    // the notification stream. EXPECTED RED today — learning a sender mutates the
    // view without a MembershipChanged; the actor must route it through the stream.
    let now = Instant::now();
    let mut a = SwimNode::new(id(0), behavioral_config(t(0)), now);
    let mut stream: Vec<(NodeId, MemberState, u64)> = Vec::new();
    for act in a.handle_ping(id(5), 1, &[]) {
        if let NodeAction::MembershipChanged { node_id, state, incarnation } = act {
            stream.push((node_id, state, incarnation));
        }
    }
    let view: View = a
        .members()
        .all_members()
        .iter()
        .map(|e| (e.node_id.0, (e.state, e.incarnation)))
        .collect();
    assert_eq!(
        fold_stream(&stream),
        view,
        "the stream must reconstruct the view even for a peer learned via its Ping (§10.2)"
    );
}

#[test]
fn goal7_no_duplicate_or_coalesced_notifications() {
    // §6.3: one notification per real change — no dupes, no coalescing. Each
    // notification for a given peer must strictly advance (dominate) the previous
    // one for that peer; an identical repeat would be a spurious duplicate.
    let mut c = Cluster::new(4, behavioral_config(t(0)));
    assert!(c.run_until(&[], 500, |c| c.all_converged_alive()), "cluster did not converge");
    for observer in 0..4 {
        let mut last: BTreeMap<[u8; 32], (MemberState, u64)> = BTreeMap::new();
        for &(nid, st, inc) in &c.notifications[observer] {
            if let Some(&(pst, pinc)) = last.get(&nid.0) {
                let advances = inc > pinc || (inc == pinc && st.priority() > pst.priority());
                assert!(
                    advances,
                    "node {observer}: a notification ({st:?},{inc}) did not advance past ({pst:?},{pinc}) — duplicate/coalesced"
                );
            }
            last.insert(nid.0, (st, inc));
        }
    }
}