diff --git a/crates/simulation/tests/cluster_scenarios.rs b/crates/simulation/tests/cluster_scenarios.rs index e858963..82063e1 100644 --- a/crates/simulation/tests/cluster_scenarios.rs +++ b/crates/simulation/tests/cluster_scenarios.rs @@ -138,9 +138,9 @@ fn cluster_converges_under_10_percent_message_loss() { swim: distribution::swim::probe::SwimConfig { probe_interval: 1, probe_timeout: 5, - indirect_probes: 2, - suspicion_timeout: 20, - dead_reprobe_interval: 30, + indirect_probes: 3, + suspicion_timeout: 60, + dead_reprobe_interval: 15, }, network_faults: vec![NetworkFault::SetDropRate { round: 1, diff --git a/crates/simulation/tests/debug_registry.rs b/crates/simulation/tests/debug_registry.rs deleted file mode 100644 index 3ace815..0000000 --- a/crates/simulation/tests/debug_registry.rs +++ /dev/null @@ -1,56 +0,0 @@ -use simulation::distribution::sim::{ - run_simulation_with_nodes, DistributionSimConfig, SimAction, -}; - -#[test] -fn debug_tombstone_propagation() { - let config = DistributionSimConfig { - name: "debug-tombstone".into(), - num_nodes: 5, - num_rounds: 80, - ticks_per_round: 3, - actors_per_node: 0, - action_schedule: vec![ - (5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }), - ], - kill_schedule: vec![(15, 0)], - ..DistributionSimConfig::default() - }; - - let (trace, nodes) = run_simulation_with_nodes(config); - - // Check snapshots at various rounds - for round_idx in [4, 9, 14, 19, 24, 39, 59, 79] { - if round_idx < trace.snapshots_per_round.len() { - let snaps = &trace.snapshots_per_round[round_idx]; - let reg_sizes: Vec = snaps.iter().map(|(_, s)| s.registry_size).collect(); - let tomb_counts: Vec = snaps.iter().map(|(_, s)| s.registry_tombstone_count).collect(); - let alive: Vec = snaps.iter().map(|(_, s)| s.is_alive).collect(); - let members: Vec = snaps.iter().map(|(_, s)| s.member_count).collect(); - eprintln!("Round {}: alive={alive:?} members={members:?} registry={reg_sizes:?} tombstones={tomb_counts:?}", round_idx + 1); - } - } - - // Check membership change events - let dead_events: Vec<_> = trace.events.iter() - .filter(|e| matches!(&e.kind, simulation::distribution::trace::DistributionEventKind::MembershipChanged { new_state, .. } if new_state == "Dead")) - .collect(); - eprintln!("Dead events: {}", dead_events.len()); - for e in &dead_events { - if let simulation::distribution::trace::DistributionEventKind::MembershipChanged { target, new_state } = &e.kind { - eprintln!(" tick={} node={} declared {target} as {new_state}", e.tick, e.node_name); - } - } - - // Check final nodes' resolve_name - for (i, node) in nodes.iter().enumerate() { - match node { - Some(n) => { - let res = n.resolve_name("svc"); - eprintln!("Node {i}: resolve_name('svc') = {res:?}, registry_size={}, tombstones={}", - n.registry().len(), n.registry().tombstone_count()); - } - None => eprintln!("Node {i}: DEAD"), - } - } -} diff --git a/crates/simulation/tests/distribution_lifecycle.rs b/crates/simulation/tests/distribution_lifecycle.rs new file mode 100644 index 0000000..fb9f31f --- /dev/null +++ b/crates/simulation/tests/distribution_lifecycle.rs @@ -0,0 +1,237 @@ +//! Lifecycle simulation tests — death/repair/cache/routing behavior. +//! +//! Tests that node death correctly triggers: +//! - Repair queue population for re-replication +//! - Cache invalidation of stale entries +//! - Routing table cleanup +//! - Recovery after partition heals + +use simulation::distribution::properties::{ + check_repair_queue_populated, check_routing_table_bounded, +}; +use simulation::distribution::sim::{ + run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, +}; + +fn default_config() -> DistributionSimConfig { + DistributionSimConfig::default() +} + +// ──────────────────────────────────────────────────────────────────────────── +// 1. Dead node's actors populate repair queue and invalidate cache +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn dead_node_triggers_repair_queue_and_cache_invalidation() { + // Given: 5-node cluster, 2 actors/node. Node 2 is killed at round 10. + let config = DistributionSimConfig { + name: "death-repair-cache".into(), + num_nodes: 5, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 2, + kill_schedule: vec![(10, 2)], + ..default_config() + }; + + let (trace, nodes) = run_simulation_with_nodes(config); + + // Then: at least one survivor should have a non-empty repair queue + let result = check_repair_queue_populated(&trace, 10); + assert!( + result.passed, + "repair queue should be populated after node death: {}", + result.actual + ); + + // And: no survivor's cache should contain entries pointing to the dead node + let dead_node_id = { + // Find the node_id for node 2 from round snapshots before death + // We can check from the surviving nodes + // Node 2 is dead (None), so we check survivors' caches + let mut stale_count = 0; + for node in nodes.iter().filter_map(|n| n.as_ref()) { + for (_actor, cached_on) in node.cache().entries() { + // The dead node's entries should have been invalidated + // We can't easily get node 2's ID here, but we can check + // that no survivor caches an actor on a node not in their members + let alive_ids: Vec<_> = node.members().iter().map(|m| m.node_id).collect(); + if !alive_ids.contains(&cached_on) && cached_on != node.node_id() { + stale_count += 1; + } + } + } + stale_count + }; + + assert_eq!( + dead_node_id, 0, + "no survivor should have cache entries pointing to non-member nodes" + ); +} + +// ──────────────────────────────────────────────────────────────────────────── +// 2. Revived node starts fresh (empty directory) +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn revived_node_has_empty_directory() { + // Given: 5-node cluster, 2 actors/node. + // Node 2 killed at round 10, revived at round 50. + let config = DistributionSimConfig { + name: "revive-fresh".into(), + num_nodes: 5, + num_rounds: 100, + ticks_per_round: 3, + actors_per_node: 2, + kill_schedule: vec![(10, 2)], + revive_schedule: vec![(50, 2)], + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // Then: the revived node should have an empty directory + // (it's a fresh DistributedNode, not carrying over old state) + let revived = nodes[2].as_ref().expect("node 2 should be revived"); + assert_eq!( + revived.directory().entry_count(), 0, + "revived node should start with empty directory" + ); + + // And: the revived node should have rejoined the cluster + assert!( + !revived.members().is_empty(), + "revived node should have some cluster members" + ); +} + +// ──────────────────────────────────────────────────────────────────────────── +// 3. Cache invalidation tracks membership changes +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn cache_shrinks_after_node_death() { + // Given: 5-node cluster with actors, all caches populated during setup. + // When: node 1 is killed + // Then: cache_size should decrease for survivors after death detection. + let config = DistributionSimConfig { + name: "cache-invalidation".into(), + num_nodes: 5, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 3, + kill_schedule: vec![(10, 1)], + ..default_config() + }; + + let (trace, _nodes) = run_simulation_with_nodes(config); + + // Check that cache_size decreased for at least some survivors after death + // Before death (round 9), survivors should have cache entries + // After death detection, cache for dead node's actors should be invalidated + let pre_death_round = 8; // 0-indexed round 9 + let post_detection_round = 39; // well after SWIM detection + + if pre_death_round < trace.snapshots_per_round.len() + && post_detection_round < trace.snapshots_per_round.len() + { + let pre_cache_max: usize = trace.snapshots_per_round[pre_death_round] + .iter() + .filter(|(_, s)| s.is_alive) + .map(|(_, s)| s.cache_size) + .max() + .unwrap_or(0); + + let post_cache_sizes: Vec = trace.snapshots_per_round[post_detection_round] + .iter() + .filter(|(_, s)| s.is_alive) + .map(|(_, s)| s.cache_size) + .collect(); + + // After death, some survivors should have fewer cache entries + // (the dead node's actors were invalidated) + let any_decreased = post_cache_sizes.iter().any(|&s| s < pre_cache_max); + assert!( + any_decreased || pre_cache_max == 0, + "cache should shrink after node death, pre_max={pre_cache_max}, post={post_cache_sizes:?}" + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 4. Routing table recovers after partition heals (dead reprobe) +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn routing_table_recovers_after_partition_heals() { + // Given: 6-node cluster, partition at round 10, heal at round 40 + // With dead_reprobe_interval=10, nodes re-discover dead members + let config = DistributionSimConfig { + name: "rt-recovery".into(), + num_nodes: 6, + num_rounds: 120, + ticks_per_round: 3, + actors_per_node: 0, + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 1, + suspicion_timeout: 5, + dead_reprobe_interval: 10, + }, + network_faults: vec![ + NetworkFault::Partition { + round: 10, + partition: Partition { + side_a: vec![0, 1, 2], + side_b: vec![3, 4, 5], + asymmetric: false, + }, + }, + NetworkFault::Heal { round: 40 }, + ], + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // Then: after healing, all nodes should have recovered routing tables + // Each node should see at least 4 of 5 other nodes in their routing table + for (i, maybe_node) in nodes.iter().enumerate() { + if let Some(node) = maybe_node { + assert!( + node.routing_table().len() >= 4, + "node {i} should have ≥4 RT entries after partition heals, got {}", + node.routing_table().len() + ); + } + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 5. Routing table tracks alive membership (bounded invariant) +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn routing_table_bounded_by_alive_count() { + // Run a simulation with deaths and verify the routing table invariant + let config = DistributionSimConfig { + name: "rt-bounded".into(), + num_nodes: 8, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 1, + kill_schedule: vec![(15, 2), (25, 5)], + ..default_config() + }; + + let (trace, _) = run_simulation_with_nodes(config); + + let result = check_routing_table_bounded(&trace); + assert!( + result.passed, + "routing table should never exceed alive count: {}", + result.actual + ); +} diff --git a/crates/simulation/tests/distribution_properties.rs b/crates/simulation/tests/distribution_properties.rs new file mode 100644 index 0000000..6e46385 --- /dev/null +++ b/crates/simulation/tests/distribution_properties.rs @@ -0,0 +1,260 @@ +//! Property-based distribution tests — invariants that must hold across configs. +//! +//! Each test verifies a structural property across multiple simulation +//! configurations with varying fault conditions. + +use simulation::distribution::properties::{ + check_cache_bounded, check_registry_propagation, check_repair_queue_populated, + check_routing_table_bounded, +}; +use simulation::distribution::sim::{ + run_simulation_with_nodes, DistributionSimConfig, SimAction, +}; + +// ──────────────────────────────────────────────────────────────────────────── +// 1. Routing table size ≤ alive membership at every round +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn routing_table_bounded_across_configs() { + let configs = vec![ + // Healthy 5-node cluster + DistributionSimConfig { + name: "rt-bound-healthy-5".into(), + num_nodes: 5, + num_rounds: 50, + ticks_per_round: 3, + ..DistributionSimConfig::default() + }, + // 10-node cluster with 2 deaths + DistributionSimConfig { + name: "rt-bound-deaths-10".into(), + num_nodes: 10, + num_rounds: 60, + ticks_per_round: 3, + kill_schedule: vec![(15, 3), (25, 7)], + ..DistributionSimConfig::default() + }, + // 15-node cluster with 1 death + DistributionSimConfig { + name: "rt-bound-large-15".into(), + num_nodes: 15, + num_rounds: 60, + ticks_per_round: 3, + kill_schedule: vec![(20, 0)], + ..DistributionSimConfig::default() + }, + ]; + + for config in configs { + let name = config.name.clone(); + let (trace, _) = run_simulation_with_nodes(config); + let result = check_routing_table_bounded(&trace); + assert!( + result.passed, + "[{name}] routing table bounded invariant violated: {}", + result.actual + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 2. Cache size ≤ cache_capacity at every round +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn cache_bounded_across_configs() { + let capacity = 100; + let configs = vec![ + DistributionSimConfig { + name: "cache-bound-5".into(), + num_nodes: 5, + num_rounds: 50, + ticks_per_round: 3, + actors_per_node: 5, + cache_capacity: capacity, + ..DistributionSimConfig::default() + }, + DistributionSimConfig { + name: "cache-bound-10-deaths".into(), + num_nodes: 10, + num_rounds: 60, + ticks_per_round: 3, + actors_per_node: 3, + cache_capacity: capacity, + kill_schedule: vec![(15, 2), (20, 5)], + ..DistributionSimConfig::default() + }, + ]; + + for config in configs { + let name = config.name.clone(); + let (trace, _) = run_simulation_with_nodes(config); + let result = check_cache_bounded(&trace, capacity); + assert!( + result.passed, + "[{name}] cache bounded invariant violated: {}", + result.actual + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 3. Repair queue populates when node with directory entries dies +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn repair_queue_populates_on_death_with_directory_entries() { + // Node 2 has 3 actors. When it dies, repair queue should grow. + let config = DistributionSimConfig { + name: "repair-proportional".into(), + num_nodes: 5, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 3, + kill_schedule: vec![(15, 2)], + ..DistributionSimConfig::default() + }; + + let (trace, _) = run_simulation_with_nodes(config); + + // Repair queue should be populated within 2-3 rounds of death detection + let result = check_repair_queue_populated(&trace, 15); + assert!( + result.passed, + "repair queue should fill after node with actors dies: {}", + result.actual + ); + + // Check that the total repair queue size across survivors is proportional + // to the dead node's directory entries (3 actors) + let post_death_sizes: Vec = trace + .snapshots_per_round + .iter() + .skip(20) + .take(30) + .flat_map(|round_snaps| { + round_snaps + .iter() + .filter(|(_, s)| s.is_alive) + .map(|(_, s)| s.repair_queue_size) + }) + .collect(); + + let max_repair = post_death_sizes.iter().max().copied().unwrap_or(0); + assert!( + max_repair >= 1, + "at least one repair queue entry expected for dead node's actors, max seen: {max_repair}" + ); +} + +// ──────────────────────────────────────────────────────────────────────────── +// 4. Registry convergence — eventual consistency across configs +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn registry_eventually_consistent_across_configs() { + let configs = vec![ + // 5 nodes, 1 name + ( + DistributionSimConfig { + name: "reg-ec-simple".into(), + num_nodes: 5, + num_rounds: 50, + ticks_per_round: 3, + actors_per_node: 0, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "svc-a".into() }), + ], + ..DistributionSimConfig::default() + }, + 1, // expected min registry size + ), + // 8 nodes, 3 names from different nodes + ( + DistributionSimConfig { + name: "reg-ec-multi".into(), + num_nodes: 8, + num_rounds: 60, + ticks_per_round: 3, + actors_per_node: 0, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }), + (5, SimAction::RegisterName { node_idx: 3, name: "beta".into() }), + (5, SimAction::RegisterName { node_idx: 6, name: "gamma".into() }), + ], + ..DistributionSimConfig::default() + }, + 3, + ), + // 5 nodes, register + kill owner, verify tombstone propagates + ( + DistributionSimConfig { + name: "reg-ec-death".into(), + num_nodes: 5, + num_rounds: 80, + ticks_per_round: 3, + actors_per_node: 0, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "ephemeral".into() }), + ], + kill_schedule: vec![(15, 0)], + ..DistributionSimConfig::default() + }, + 1, // tombstoned entry still counts as registry_size + ), + ]; + + for (config, min_size) in configs { + let name = config.name.clone(); + let (trace, _) = run_simulation_with_nodes(config); + let result = check_registry_propagation(&trace, min_size); + assert!( + result.passed, + "[{name}] registry should converge: {}", + result.actual + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 5. Multiple deaths don't violate invariants +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn cascading_deaths_maintain_invariants() { + // 8 nodes, 3 die in sequence + let config = DistributionSimConfig { + name: "cascade-invariants".into(), + num_nodes: 8, + num_rounds: 100, + ticks_per_round: 3, + actors_per_node: 2, + cache_capacity: 200, + kill_schedule: vec![(10, 1), (20, 3), (30, 5)], + ..DistributionSimConfig::default() + }; + + let (trace, _) = run_simulation_with_nodes(config); + + let rt_result = check_routing_table_bounded(&trace); + assert!( + rt_result.passed, + "routing table bounded after cascading deaths: {}", + rt_result.actual + ); + + let cache_result = check_cache_bounded(&trace, 200); + assert!( + cache_result.passed, + "cache bounded after cascading deaths: {}", + cache_result.actual + ); + + let repair_result = check_repair_queue_populated(&trace, 10); + assert!( + repair_result.passed, + "repair queue populated after first death: {}", + repair_result.actual + ); +}