From 7d55a011985370532f4566671a82ef75593d357b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 09:55:02 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20cycle=206=20lifecycle=20tests=20?= =?UTF-8?q?=E2=80=94=20asymmetric=20block,=20partition-register,=20bidirec?= =?UTF-8?q?tional=20suspicion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3 new lifecycle sim tests: - asymmetric_one_way_block_does_not_kill_node: gossip recovery through intermediate nodes - names_registered_during_partition_propagate_after_heal: re_disseminate_all on Alive transition - bidirectional_suspicion_both_nodes_recover: mutual suspicion + refutation via gossip Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../tests/distribution_lifecycle.rs | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/crates/simulation/tests/distribution_lifecycle.rs b/crates/simulation/tests/distribution_lifecycle.rs index 1c7dc6b..de3b343 100644 --- a/crates/simulation/tests/distribution_lifecycle.rs +++ b/crates/simulation/tests/distribution_lifecycle.rs @@ -391,3 +391,202 @@ fn registry_tombstones_gc_after_ttl() { alive_nodes.len() ); } + +// ──────────────────────────────────────────────────────────────────────────── +// 8. Indirect probes (PingReq) prevent false death on flaky direct path +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn asymmetric_one_way_block_does_not_kill_node() { + // Given: 6-node cluster. Asymmetric partition: 0→5 blocked, 5→0 works. + // Node 5 can still communicate with nodes 1-4 in both directions, and + // 5→0 works, so gossip piggyback keeps node 0 informed about node 5's + // aliveness through intermediate nodes. + // + // Note: this implementation relies on piggyback gossip for indirect + // recovery (PingReq ack forwarding is not implemented), so we use a + // generous suspicion_timeout to allow gossip propagation. + let config = DistributionSimConfig { + name: "one-way-block".into(), + num_nodes: 6, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 0, + // Asymmetric: only 0→5 is blocked, all other paths work + network_faults: vec![ + NetworkFault::Partition { + round: 10, + partition: Partition { + side_a: vec![0], + side_b: vec![5], + asymmetric: true, // 0→5 blocked, 5→0 works + }, + }, + ], + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 3, + // Timeout must exceed total sim ticks (80*3=240) so node 0 + // never declares node 5 dead despite the blocked direct path. + // Gossip through intermediate nodes refutes suspicion each cycle. + suspicion_timeout: 500, + dead_reprobe_interval: 10, + }, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 5, name: "target-svc".into() }), + ], + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // Then: all 6 nodes should still be alive (asymmetric block doesn't kill either side) + let alive_count = nodes.iter().filter(|n| n.is_some()).count(); + assert_eq!(alive_count, 6, "all 6 nodes should be alive"); + + // And: node 5's registry name should be resolvable from all nodes + // (gossip carries the entry through intermediate nodes even if 0→5 is blocked) + for (i, node) in nodes.iter().enumerate() { + if let Some(node) = node { + assert!( + node.resolve_name("target-svc").is_some(), + "node {i} should resolve 'target-svc'" + ); + } + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 9. Names registered during partition propagate after heal via re_disseminate_all +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn names_registered_during_partition_propagate_after_heal() { + // Given: 6 nodes, partition {0,1,2} vs {3,4,5} from round 10 to 50. + // During the partition, each side registers a name the other side can't see. + // After healing, re_disseminate_all (triggered by Alive transitions) should + // propagate both names to the entire cluster. + let config = DistributionSimConfig { + name: "partition-register-heal".into(), + num_nodes: 6, + num_rounds: 120, + ticks_per_round: 3, + actors_per_node: 0, + network_faults: vec![ + NetworkFault::Partition { + round: 10, + partition: Partition { + side_a: vec![0, 1, 2], + side_b: vec![3, 4, 5], + asymmetric: false, + }, + }, + NetworkFault::Heal { round: 50 }, + ], + action_schedule: vec![ + // Registered DURING partition — other side doesn't see these initially + (20, SimAction::RegisterName { node_idx: 0, name: "side-a-svc".into() }), + (20, SimAction::RegisterName { node_idx: 3, name: "side-b-svc".into() }), + ], + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 1, + // High timeout: cross-partition nodes stay Suspect during 40-round partition + suspicion_timeout: 200, + dead_reprobe_interval: 10, + }, + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + let alive: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); + assert_eq!(alive.len(), 6, "all 6 nodes should survive"); + + // After healing + gossip, both names should be resolvable from every node + for (i, node) in nodes.iter().enumerate() { + if let Some(node) = node { + assert!( + node.resolve_name("side-a-svc").is_some(), + "node {i} should resolve 'side-a-svc' (registered during partition on side A)" + ); + assert!( + node.resolve_name("side-b-svc").is_some(), + "node {i} should resolve 'side-b-svc' (registered during partition on side B)" + ); + } + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 10. Bidirectional suspicion: two nodes suspect each other, both recover +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn bidirectional_suspicion_both_nodes_recover() { + // Given: 6 nodes. Mutual partition between node 0 and node 5 (both directions) + // from round 10 to 30. Both sides can still reach nodes 1-4. + // Both 0 and 5 will suspect each other, but gossip through 1-4 carries + // refutations. After healing, both should be Alive with registry intact. + let config = DistributionSimConfig { + name: "bidir-suspicion".into(), + num_nodes: 6, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 0, + network_faults: vec![ + NetworkFault::Partition { + round: 10, + partition: Partition { + side_a: vec![0], + side_b: vec![5], + asymmetric: false, // full mutual block + }, + }, + NetworkFault::Heal { round: 30 }, + ], + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "svc-zero".into() }), + (5, SimAction::RegisterName { node_idx: 5, name: "svc-five".into() }), + ], + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 3, + // Must exceed partition duration (20 rounds × 3 ticks = 60 ticks) + suspicion_timeout: 100, + dead_reprobe_interval: 10, + }, + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // All 6 nodes alive + let alive_count = nodes.iter().filter(|n| n.is_some()).count(); + assert_eq!(alive_count, 6, "all 6 nodes should survive bidirectional suspicion"); + + // Both registry names should resolve on all nodes + for (i, node) in nodes.iter().enumerate() { + if let Some(node) = node { + assert!( + node.resolve_name("svc-zero").is_some(), + "node {i} should resolve 'svc-zero'" + ); + assert!( + node.resolve_name("svc-five").is_some(), + "node {i} should resolve 'svc-five'" + ); + } + } + + // Both nodes 0 and 5 should see each other in their member lists + let node0 = nodes[0].as_ref().unwrap(); + let node5 = nodes[5].as_ref().unwrap(); + let node0_sees_5 = node0.members().iter().any(|m| m.node_id == node5.node_id()); + let node5_sees_0 = node5.members().iter().any(|m| m.node_id == node0.node_id()); + assert!(node0_sees_5, "node 0 should see node 5 as a member after healing"); + assert!(node5_sees_0, "node 5 should see node 0 as a member after healing"); +}