Zachery Aaron Shores-Chmielewski
3fd7c624bd
Reduce idle cpu usage on my main machine from 17% to 1%. Made SWIM gossip more lazy. Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
1003 lines
39 KiB
Rust
1003 lines
39 KiB
Rust
#![cfg(feature = "distribution")]
|
||
//! Cluster simulation scenarios — breadth-first coverage of failure modes.
|
||
//!
|
||
//! Inspired by Hashicorp memberlist test suite, FoundationDB simulation testing,
|
||
//! and Jepsen/Antithesis fault injection patterns.
|
||
|
||
use simulation::distribution::properties::{
|
||
analyze, check_accuracy, check_completeness, check_convergence, check_failure_detection,
|
||
check_membership_accuracy,
|
||
};
|
||
use simulation::distribution::sim::{
|
||
run_simulation, DistributionSimConfig, NetworkFault, Partition,
|
||
};
|
||
use simulation::distribution::trace::DistributionEventKind;
|
||
|
||
fn default_config() -> DistributionSimConfig {
|
||
DistributionSimConfig::default()
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 1. Network Partition — symmetric split-brain
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn symmetric_partition_splits_membership_views() {
|
||
// Given: 6 nodes, partition {0,1,2} vs {3,4,5} at round 10
|
||
// Long partitions cause SWIM to declare the other side dead — this is correct behavior.
|
||
// SWIM does not auto-rediscover dead nodes after partition heals.
|
||
let config = DistributionSimConfig {
|
||
name: "symmetric-partition".into(),
|
||
num_nodes: 6,
|
||
num_rounds: 60,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1, 2],
|
||
side_b: vec![3, 4, 5],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// After partition, each side should form its own sub-cluster.
|
||
// Each side of 3 nodes should see exactly 2 other members (its own group).
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
|
||
// Side A nodes (0,1,2) should see ≤2 members each (only their partition)
|
||
for idx in 0..3 {
|
||
let snap = &last_round[idx].1;
|
||
assert!(
|
||
snap.is_alive && snap.member_count <= 3,
|
||
"side_a node {} sees {} members, expected ≤3",
|
||
idx,
|
||
snap.member_count
|
||
);
|
||
}
|
||
|
||
// Side B nodes (3,4,5) should also see ≤2 members
|
||
for idx in 3..6 {
|
||
let snap = &last_round[idx].1;
|
||
assert!(
|
||
snap.is_alive && snap.member_count <= 3,
|
||
"side_b node {} sees {} members, expected ≤3",
|
||
idx,
|
||
snap.member_count
|
||
);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 2. Asymmetric partition — one-way communication failure
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn asymmetric_partition_causes_one_sided_suspicion() {
|
||
// Given: 5 nodes, node 4 can send to 0 but 0 can't send to 4
|
||
let config = DistributionSimConfig {
|
||
name: "asymmetric-partition".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0],
|
||
side_b: vec![4],
|
||
asymmetric: true, // 0→4 blocked, 4→0 works
|
||
},
|
||
},
|
||
NetworkFault::Heal { round: 50 },
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// After healing, the cluster should eventually recover
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let alive_with_members: Vec<_> = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive)
|
||
.map(|(_, s)| s.member_count)
|
||
.collect();
|
||
|
||
// All nodes should see at least 3 members after healing
|
||
assert!(
|
||
alive_with_members.iter().all(|&c| c >= 3),
|
||
"all nodes should recover after asymmetric partition heals, got: {alive_with_members:?}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 3. Message loss — random packet dropping
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn cluster_converges_under_10_percent_message_loss() {
|
||
// Given: 5 nodes with 10% message loss from the start.
|
||
// 10% loss is significant for SWIM because it can hit both direct probe
|
||
// AND indirect probes in the same cycle, causing false suspicions.
|
||
// Dead reprobe is enabled so false deaths can self-correct — without it,
|
||
// correct death dissemination (via piggyback) causes cascading false deaths
|
||
// that collapse the entire cluster under even modest message loss.
|
||
let config = DistributionSimConfig {
|
||
name: "message-loss-10pct".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 150,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 5,
|
||
indirect_probes: 3,
|
||
suspicion_timeout: 60,
|
||
dead_reprobe_interval: 15,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![NetworkFault::SetDropRate {
|
||
round: 1,
|
||
rate: 0.1,
|
||
}],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
let metrics = analyze(&trace);
|
||
|
||
// With 10% loss, SWIM's probe cycle is disrupted enough to cause
|
||
// false suspicions. Dead reprobe allows recovery. We verify:
|
||
// 1. The simulation completes without panic (implicit — we got here)
|
||
// 2. At least partial membership is maintained (some nodes still know about others)
|
||
let result = check_membership_accuracy(&metrics, 0.15);
|
||
assert!(
|
||
result.passed,
|
||
"cluster should maintain some membership under 10% loss: {}",
|
||
result.actual
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn heavy_message_loss_causes_membership_instability() {
|
||
// Given: 5 nodes with 30% message loss
|
||
// Heavy loss overwhelms SWIM's probe cycle, causing false suspicions.
|
||
// This tests that the protocol degrades but doesn't crash.
|
||
let config = DistributionSimConfig {
|
||
name: "message-loss-30pct".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 5,
|
||
indirect_probes: 2,
|
||
suspicion_timeout: 15,
|
||
dead_reprobe_interval: 0,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![NetworkFault::SetDropRate {
|
||
round: 1,
|
||
rate: 0.3,
|
||
}],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// The simulation should complete without panicking.
|
||
// Under heavy loss, some membership instability is expected.
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let alive_count = last_round.iter().filter(|(_, s)| s.is_alive).count();
|
||
assert_eq!(alive_count, 5, "no nodes should actually die");
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 4. Seed node failure — cluster survives without the seed
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn cluster_survives_seed_node_death() {
|
||
// Given: 5 nodes, kill the seed (node 0) at round 15
|
||
let config = DistributionSimConfig {
|
||
name: "seed-death".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![(15, 0)], // Kill the seed!
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// Surviving 4 nodes should detect the seed's death
|
||
let result = check_failure_detection(&trace, 4);
|
||
assert!(
|
||
result.passed,
|
||
"survivors should detect seed death: {}",
|
||
result.actual
|
||
);
|
||
|
||
// Survivors should still maintain membership among themselves
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let survivors: Vec<_> = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive)
|
||
.collect();
|
||
assert_eq!(survivors.len(), 4, "4 survivors expected");
|
||
|
||
// At least 3 of 4 survivors should see each other
|
||
let well_connected = survivors
|
||
.iter()
|
||
.filter(|(_, s)| s.member_count >= 2)
|
||
.count();
|
||
assert!(
|
||
well_connected >= 3,
|
||
"at least 3 survivors should see ≥2 members, got {well_connected}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 5. Simultaneous multi-node failure
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn simultaneous_two_node_failure_detected() {
|
||
// Given: 7 nodes, kill nodes 2 and 5 simultaneously at round 15
|
||
let config = DistributionSimConfig {
|
||
name: "multi-kill".into(),
|
||
num_nodes: 7,
|
||
num_rounds: 100,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![(15, 2), (15, 5)],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// 5 survivors should see at most 5 members (detecting both deaths)
|
||
let result = check_failure_detection(&trace, 5);
|
||
assert!(
|
||
result.passed,
|
||
"survivors should detect both deaths: {}",
|
||
result.actual
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 6. Cascading sequential failure
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn cascading_failures_leave_quorum_intact() {
|
||
// Given: 7 nodes, kill one at r=10, another at r=25, another at r=40
|
||
let config = DistributionSimConfig {
|
||
name: "cascading-failure".into(),
|
||
num_nodes: 7,
|
||
num_rounds: 100,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![(10, 1), (25, 3), (40, 5)],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// 4 survivors should still form a connected cluster
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let survivors: Vec<_> = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive)
|
||
.collect();
|
||
assert_eq!(survivors.len(), 4, "4 survivors expected");
|
||
|
||
// Each survivor should see at most 4 members
|
||
for (name, snap) in &survivors {
|
||
assert!(
|
||
snap.member_count <= 4,
|
||
"{name} sees {} members, expected ≤4",
|
||
snap.member_count
|
||
);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 7. Large cluster convergence
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn cluster_of_fifty_converges() {
|
||
// Given: 50 nodes
|
||
let config = DistributionSimConfig {
|
||
name: "fifty-converges".into(),
|
||
num_nodes: 50,
|
||
num_rounds: 150,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 2,
|
||
suspicion_timeout: 10,
|
||
dead_reprobe_interval: 0,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
let metrics = analyze(&trace);
|
||
|
||
// 50-node cluster should reach ≥80% accuracy
|
||
let result = check_membership_accuracy(&metrics, 0.8);
|
||
assert!(
|
||
result.passed,
|
||
"50-node cluster should converge: {}",
|
||
result.actual
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 8. Rapid churn — nodes joining and dying frequently
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn rapid_churn_maintains_partial_membership() {
|
||
// Given: 8 nodes with rapid kill/revive cycles
|
||
let config = DistributionSimConfig {
|
||
name: "rapid-churn".into(),
|
||
num_nodes: 8,
|
||
num_rounds: 100,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![
|
||
(10, 2),
|
||
(15, 4),
|
||
(30, 6),
|
||
(45, 3),
|
||
],
|
||
revive_schedule: vec![
|
||
(25, 2),
|
||
(35, 4),
|
||
(55, 6),
|
||
(65, 3),
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// At end, all nodes should be alive and have some membership view
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let alive_count = last_round.iter().filter(|(_, s)| s.is_alive).count();
|
||
assert_eq!(alive_count, 8, "all 8 nodes should be alive at end");
|
||
|
||
// At least half should have reasonable membership
|
||
let connected = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive && s.member_count >= 3)
|
||
.count();
|
||
assert!(
|
||
connected >= 4,
|
||
"at least 4 of 8 nodes should see ≥3 members after churn, got {connected}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 9. Graceful leave — node announces departure
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn graceful_leave_detected_faster_than_crash() {
|
||
// We can't directly test graceful leave in the current sim harness
|
||
// (leave() is called but doesn't disseminate through ticks in the same way).
|
||
// Instead, test that a crash is detected within a bounded number of rounds.
|
||
|
||
// Given: 5 nodes, kill node 2 at round 5
|
||
let config = DistributionSimConfig {
|
||
name: "crash-detection-speed".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 40,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![(5, 2)],
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 2,
|
||
indirect_probes: 1,
|
||
suspicion_timeout: 5,
|
||
dead_reprobe_interval: 0,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// Death should be detected by round 20 (suspicion_timeout + margin)
|
||
let mut detected_by_round = None;
|
||
for (round_idx, round_snaps) in trace.snapshots_per_round.iter().enumerate() {
|
||
if round_idx < 5 {
|
||
continue; // Skip rounds before kill
|
||
}
|
||
let all_survivors_see_reduced = round_snaps
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive)
|
||
.all(|(_, s)| s.member_count <= 4);
|
||
if all_survivors_see_reduced {
|
||
detected_by_round = Some(round_idx + 1);
|
||
break;
|
||
}
|
||
}
|
||
|
||
assert!(
|
||
detected_by_round.is_some(),
|
||
"crash should be detected before end of simulation"
|
||
);
|
||
let round = detected_by_round.unwrap();
|
||
assert!(
|
||
round <= 25,
|
||
"crash should be detected by round 25, was detected at round {round}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 10. Partition then kill — compounding failures
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn partition_plus_kill_in_minority_side() {
|
||
// Given: 5 nodes, partition {0,1,2} vs {3,4}, then kill node 3
|
||
let config = DistributionSimConfig {
|
||
name: "partition-plus-kill".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 100,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1, 2],
|
||
side_b: vec![3, 4],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
NetworkFault::Heal { round: 70 },
|
||
],
|
||
kill_schedule: vec![(20, 3)], // Kill in minority side
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// After healing, 4 alive nodes should re-converge
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let alive_nodes: Vec<_> = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive)
|
||
.collect();
|
||
assert_eq!(alive_nodes.len(), 4, "4 nodes should be alive");
|
||
|
||
// Majority side {0,1,2} should be well-connected
|
||
let majority_connected = last_round[..3]
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive && s.member_count >= 2)
|
||
.count();
|
||
assert!(
|
||
majority_connected >= 2,
|
||
"majority partition should maintain connectivity"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 11. Actor resolution under network faults
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn actor_resolution_degrades_during_partition() {
|
||
// Given: 5 nodes with actors, partition at round 10
|
||
let config = DistributionSimConfig {
|
||
name: "actor-resolution-partition".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 2,
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1],
|
||
side_b: vec![2, 3, 4],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
NetworkFault::Heal { round: 50 },
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// Should still have some successful resolutions (from cache)
|
||
let resolve_events: Vec<_> = trace
|
||
.events
|
||
.iter()
|
||
.filter(|e| matches!(e.kind, DistributionEventKind::ActorResolved { .. }))
|
||
.collect();
|
||
|
||
assert!(
|
||
!resolve_events.is_empty(),
|
||
"should still resolve some actors (cached) during partition"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 12. Dissemination completeness — all nodes learn about membership changes
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn membership_changes_disseminate_to_all_nodes() {
|
||
// Given: 10 nodes, kill node 5 at round 20
|
||
let config = DistributionSimConfig {
|
||
name: "dissemination-completeness".into(),
|
||
num_nodes: 10,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![(20, 5)],
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 2,
|
||
suspicion_timeout: 8,
|
||
dead_reprobe_interval: 0,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// All 9 survivors should eventually detect the death
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let survivors_with_correct_view = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive && s.member_count <= 9)
|
||
.count();
|
||
|
||
assert!(
|
||
survivors_with_correct_view >= 7,
|
||
"at least 7 of 9 survivors should detect node death, got {survivors_with_correct_view}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 13. Multiple partitions in sequence
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn sequential_partitions_fragment_cluster() {
|
||
// Given: 6 nodes, two sequential partitions.
|
||
// SWIM doesn't auto-rediscover dead-declared nodes, so each partition
|
||
// permanently reduces the membership view of affected nodes.
|
||
let config = DistributionSimConfig {
|
||
name: "sequential-partitions".into(),
|
||
num_nodes: 6,
|
||
num_rounds: 100,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
network_faults: vec![
|
||
// Partition: {0,1,2} vs {3,4,5}
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1, 2],
|
||
side_b: vec![3, 4, 5],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// Each side should still see its own members
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let alive_count = last_round.iter().filter(|(_, s)| s.is_alive).count();
|
||
assert_eq!(alive_count, 6, "all nodes still alive");
|
||
|
||
// Side A should maintain internal connectivity
|
||
let side_a_connected = (0..3)
|
||
.filter(|&i| last_round[i].1.member_count >= 1)
|
||
.count();
|
||
assert!(
|
||
side_a_connected >= 2,
|
||
"at least 2 of side_a nodes should see peers, got {side_a_connected}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 14. Message loss then recovery
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn cluster_survives_brief_message_loss() {
|
||
// Given: 5 nodes with 15% loss for a brief window, then clean network.
|
||
// High suspicion timeout + dead reprobe prevents permanent false positives.
|
||
// Without dead reprobe, correct death dissemination causes cascading
|
||
// false deaths that collapse the cluster.
|
||
let config = DistributionSimConfig {
|
||
name: "brief-loss-recovery".into(),
|
||
num_nodes: 5,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 5,
|
||
indirect_probes: 3,
|
||
suspicion_timeout: 60,
|
||
dead_reprobe_interval: 15,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![
|
||
NetworkFault::SetDropRate {
|
||
round: 5,
|
||
rate: 0.15,
|
||
},
|
||
NetworkFault::SetDropRate {
|
||
round: 30,
|
||
rate: 0.0,
|
||
},
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// After loss stops, most nodes should still be in each other's member lists.
|
||
// Some nodes may have been falsely declared dead during the loss window,
|
||
// but the majority should maintain connectivity.
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let well_connected = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive && s.member_count >= 2)
|
||
.count();
|
||
assert!(
|
||
well_connected >= 2,
|
||
"at least 2 nodes should see ≥2 members after brief loss, got {well_connected}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 15. Dead-node reprobe — partition heals, dead nodes recover via reprobe
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn partition_heals_via_dead_reprobe() {
|
||
// Given: 6 nodes, partition {0,1,2} vs {3,4,5} at round 10, heal at round 40.
|
||
// With dead_reprobe_interval enabled, both sides should eventually reprobe
|
||
// the other side's dead-declared nodes, triggering incarnation refutation
|
||
// and recovering the cluster.
|
||
let config = DistributionSimConfig {
|
||
name: "dead-reprobe-recovery".into(),
|
||
num_nodes: 6,
|
||
num_rounds: 120,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 1,
|
||
suspicion_timeout: 5,
|
||
dead_reprobe_interval: 10,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1, 2],
|
||
side_b: vec![3, 4, 5],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
NetworkFault::Heal { round: 40 },
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// After healing + reprobe cycles, nodes should recover cross-partition membership.
|
||
// The reprobe fires every 10 ticks; 80 remaining rounds × 3 ticks = 240 ticks ≫ 10.
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
let well_connected = last_round
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive && s.member_count >= 4)
|
||
.count();
|
||
assert!(
|
||
well_connected >= 4,
|
||
"at least 4 of 6 nodes should recover membership after partition heals via reprobe, got {well_connected}"
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// SWIM Invariant Tests — formal property checks
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn completeness_all_survivors_detect_failure() {
|
||
// SWIM completeness: every killed node is eventually detected by ALL survivors.
|
||
let config = DistributionSimConfig {
|
||
name: "completeness".into(),
|
||
num_nodes: 7,
|
||
num_rounds: 80,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
kill_schedule: vec![(15, 3)],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
// 7 nodes originally alive, kill 1 → survivors should see member_count < 7
|
||
let result = check_completeness(&trace, 15, 7);
|
||
assert!(
|
||
result.passed,
|
||
"completeness failed: {}",
|
||
result.actual
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn accuracy_no_false_permanent_deaths() {
|
||
// SWIM accuracy: after partition heals with reprobe enabled, no alive node
|
||
// should be permanently declared dead by the majority.
|
||
let config = DistributionSimConfig {
|
||
name: "accuracy".into(),
|
||
num_nodes: 6,
|
||
num_rounds: 120,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 1,
|
||
suspicion_timeout: 5,
|
||
dead_reprobe_interval: 10,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1, 2],
|
||
side_b: vec![3, 4, 5],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
NetworkFault::Heal { round: 40 },
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
// All 6 nodes are alive. At least 80% should be well-connected.
|
||
let result = check_accuracy(&trace, 0.8);
|
||
assert!(
|
||
result.passed,
|
||
"accuracy failed: {}",
|
||
result.actual
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn convergence_after_partition_heal() {
|
||
// SWIM convergence: after partition heals, surviving nodes' member_count
|
||
// values should converge to the same value.
|
||
let config = DistributionSimConfig {
|
||
name: "convergence".into(),
|
||
num_nodes: 6,
|
||
num_rounds: 120,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 1,
|
||
suspicion_timeout: 5,
|
||
dead_reprobe_interval: 10,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0, 1, 2],
|
||
side_b: vec![3, 4, 5],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
NetworkFault::Heal { round: 40 },
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
// After round 60 (20 rounds post-heal), views should converge within ±1
|
||
let result = check_convergence(&trace, 60, 1);
|
||
assert!(
|
||
result.passed,
|
||
"convergence failed: {}",
|
||
result.actual
|
||
);
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// 16. Re-peer after partition death — stale Dead gossip must not re-kill
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn repeer_after_partition_death_stays_alive() {
|
||
// Scenario: 3 nodes. Partition isolates node 2. Node 2 gets declared Dead
|
||
// by nodes 0 and 1 (and vice versa). Partition heals. SimAction::Join
|
||
// re-peers all pairs. All nodes must converge to full alive membership.
|
||
//
|
||
// The purge_node fix ensures that clear_dead_member removes stale
|
||
// (node_id, Dead, incarnation) updates from the DisseminationQueue,
|
||
// preventing stale Dead gossip from leaking out on subsequent messages
|
||
// and re-infecting the cluster during the recovery window.
|
||
use simulation::distribution::sim::SimAction;
|
||
|
||
let config = DistributionSimConfig {
|
||
name: "repeer-after-partition-death".into(),
|
||
num_nodes: 3,
|
||
num_rounds: 100,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 1,
|
||
suspicion_timeout: 5,
|
||
dead_reprobe_interval: 10,
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![
|
||
// Isolate node 2 from nodes 0 and 1
|
||
NetworkFault::Partition {
|
||
round: 5,
|
||
partition: Partition {
|
||
side_a: vec![0, 1],
|
||
side_b: vec![2],
|
||
asymmetric: false,
|
||
},
|
||
},
|
||
// Heal the partition before the re-peer
|
||
NetworkFault::Heal { round: 20 },
|
||
],
|
||
// Re-peer all cross-partition pairs after heal
|
||
action_schedule: vec![
|
||
(25, SimAction::Join { node_idx: 0, seed_idx: 2 }),
|
||
(25, SimAction::Join { node_idx: 2, seed_idx: 0 }),
|
||
(25, SimAction::Join { node_idx: 1, seed_idx: 2 }),
|
||
(25, SimAction::Join { node_idx: 2, seed_idx: 1 }),
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// After re-peer + refutation cycles, all 3 nodes should converge.
|
||
// Check the last 10 rounds: every node must be alive and see ≥2 members.
|
||
for round_idx in (trace.num_rounds - 10)..trace.num_rounds {
|
||
let round_snaps = &trace.snapshots_per_round[round_idx];
|
||
for (name, snap) in round_snaps {
|
||
assert!(
|
||
snap.is_alive,
|
||
"round {}: {name} should be alive",
|
||
round_idx + 1
|
||
);
|
||
}
|
||
let min_members = round_snaps
|
||
.iter()
|
||
.filter(|(_, s)| s.is_alive)
|
||
.map(|(_, s)| s.member_count)
|
||
.min()
|
||
.unwrap_or(0);
|
||
assert!(
|
||
min_members >= 2,
|
||
"round {}: all nodes should see full membership (min member_count = {min_members})",
|
||
round_idx + 1
|
||
);
|
||
}
|
||
}
|
||
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
// Postmortem 3f — suspect→refute race: no false death after refutation
|
||
// ────────────────────────────────────────────────────────────────────────────
|
||
|
||
#[test]
|
||
fn suspect_refuted_before_timeout_no_false_death() {
|
||
// Scenario from SWIM flaky test postmortem (recommendation 3f):
|
||
// 3-node cluster, asymmetric partitions make node 1 unreachable by
|
||
// probes from nodes 0 and 2, but node 1's outgoing messages still
|
||
// reach them. Heal before the suspicion timer fires, so node 1 learns
|
||
// about its suspicion via gossip, bumps incarnation, and refutes back
|
||
// to Alive. The suspicion timer then fires but sees Alive (not Suspect)
|
||
// and does nothing — exercising the Bug 1 fix.
|
||
//
|
||
// Timing: partition at round 10 (tick 30), heal at round 20 (tick 60).
|
||
// Suspicion starts ~tick 36. With suspicion_timeout=50, timer fires
|
||
// ~tick 86 — 26 ticks after heal, giving ample time for refutation.
|
||
let config = DistributionSimConfig {
|
||
name: "suspect-refute-race".into(),
|
||
num_nodes: 3,
|
||
num_rounds: 60,
|
||
ticks_per_round: 3,
|
||
actors_per_node: 0,
|
||
swim: distribution::swim::probe::SwimConfig {
|
||
probe_interval: 1,
|
||
probe_timeout: 3,
|
||
indirect_probes: 1,
|
||
suspicion_timeout: 50,
|
||
dead_reprobe_interval: 0, // disabled — refutation must happen before death
|
||
probe_mode: distribution::swim::probe::ProbeMode::Periodic,
|
||
},
|
||
network_faults: vec![
|
||
// Block 0→1 (but 1→0 still works)
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![0],
|
||
side_b: vec![1],
|
||
asymmetric: true,
|
||
},
|
||
},
|
||
// Block 2→1 (but 1→2 still works)
|
||
NetworkFault::Partition {
|
||
round: 10,
|
||
partition: Partition {
|
||
side_a: vec![2],
|
||
side_b: vec![1],
|
||
asymmetric: true,
|
||
},
|
||
},
|
||
// Heal all partitions before suspicion timeout fires
|
||
NetworkFault::Heal { round: 20 },
|
||
],
|
||
..default_config()
|
||
};
|
||
|
||
let trace = run_simulation(config);
|
||
|
||
// All 3 nodes must be alive in the final snapshot — no false deaths
|
||
let last_round = trace.snapshots_per_round.last().unwrap();
|
||
for (name, snap) in last_round {
|
||
assert!(
|
||
snap.is_alive,
|
||
"{name} should be alive but was marked dead (false death after refutation)"
|
||
);
|
||
}
|
||
|
||
// 100% accuracy: every alive node sees all others as alive
|
||
let result = check_accuracy(&trace, 1.0);
|
||
assert!(
|
||
result.passed,
|
||
"all nodes should see full membership after refutation: {}",
|
||
result.actual
|
||
);
|
||
}
|