//! Lifecycle simulation tests — death/repair/cache/routing behavior. //! //! Tests that node death correctly triggers: //! - Repair queue population for re-replication //! - Cache invalidation of stale entries //! - Routing table cleanup //! - Recovery after partition heals use simulation::distribution::properties::{ check_repair_queue_populated, check_routing_table_bounded, }; use simulation::distribution::sim::{ run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, SimAction, }; fn default_config() -> DistributionSimConfig { DistributionSimConfig::default() } // ──────────────────────────────────────────────────────────────────────────── // 1. Dead node's actors populate repair queue and invalidate cache // ──────────────────────────────────────────────────────────────────────────── #[test] fn dead_node_triggers_repair_queue_and_cache_invalidation() { // Given: 5-node cluster, 2 actors/node. Node 2 is killed at round 10. let config = DistributionSimConfig { name: "death-repair-cache".into(), num_nodes: 5, num_rounds: 80, ticks_per_round: 3, actors_per_node: 2, kill_schedule: vec![(10, 2)], ..default_config() }; let (trace, nodes) = run_simulation_with_nodes(config); // Then: at least one survivor should have a non-empty repair queue let result = check_repair_queue_populated(&trace, 10); assert!( result.passed, "repair queue should be populated after node death: {}", result.actual ); // And: no survivor's cache should contain entries pointing to the dead node let dead_node_id = { // Find the node_id for node 2 from round snapshots before death // We can check from the surviving nodes // Node 2 is dead (None), so we check survivors' caches let mut stale_count = 0; for node in nodes.iter().filter_map(|n| n.as_ref()) { for (_actor, cached_on) in node.cache().entries() { // The dead node's entries should have been invalidated // We can't easily get node 2's ID here, but we can check // that no survivor caches an actor on a node not in their members let alive_ids: Vec<_> = node.members().iter().map(|m| m.node_id).collect(); if !alive_ids.contains(&cached_on) && cached_on != node.node_id() { stale_count += 1; } } } stale_count }; assert_eq!( dead_node_id, 0, "no survivor should have cache entries pointing to non-member nodes" ); } // ──────────────────────────────────────────────────────────────────────────── // 2. Revived node starts fresh (empty directory) // ──────────────────────────────────────────────────────────────────────────── #[test] fn revived_node_has_empty_directory() { // Given: 5-node cluster, 2 actors/node. // Node 2 killed at round 10, revived at round 50. let config = DistributionSimConfig { name: "revive-fresh".into(), num_nodes: 5, num_rounds: 100, ticks_per_round: 3, actors_per_node: 2, kill_schedule: vec![(10, 2)], revive_schedule: vec![(50, 2)], ..default_config() }; let (_trace, nodes) = run_simulation_with_nodes(config); // Then: the revived node should have an empty directory // (it's a fresh DistributedNode, not carrying over old state) let revived = nodes[2].as_ref().expect("node 2 should be revived"); assert_eq!( revived.directory().entry_count(), 0, "revived node should start with empty directory" ); // And: the revived node should have rejoined the cluster assert!( !revived.members().is_empty(), "revived node should have some cluster members" ); } // ──────────────────────────────────────────────────────────────────────────── // 3. Cache invalidation tracks membership changes // ──────────────────────────────────────────────────────────────────────────── #[test] fn cache_shrinks_after_node_death() { // Given: 5-node cluster with actors, all caches populated during setup. // When: node 1 is killed // Then: cache_size should decrease for survivors after death detection. let config = DistributionSimConfig { name: "cache-invalidation".into(), num_nodes: 5, num_rounds: 80, ticks_per_round: 3, actors_per_node: 3, kill_schedule: vec![(10, 1)], ..default_config() }; let (trace, _nodes) = run_simulation_with_nodes(config); // Check that cache_size decreased for at least some survivors after death // Before death (round 9), survivors should have cache entries // After death detection, cache for dead node's actors should be invalidated let pre_death_round = 8; // 0-indexed round 9 let post_detection_round = 39; // well after SWIM detection if pre_death_round < trace.snapshots_per_round.len() && post_detection_round < trace.snapshots_per_round.len() { let pre_cache_max: usize = trace.snapshots_per_round[pre_death_round] .iter() .filter(|(_, s)| s.is_alive) .map(|(_, s)| s.cache_size) .max() .unwrap_or(0); let post_cache_sizes: Vec = trace.snapshots_per_round[post_detection_round] .iter() .filter(|(_, s)| s.is_alive) .map(|(_, s)| s.cache_size) .collect(); // After death, some survivors should have fewer cache entries // (the dead node's actors were invalidated) let any_decreased = post_cache_sizes.iter().any(|&s| s < pre_cache_max); assert!( any_decreased || pre_cache_max == 0, "cache should shrink after node death, pre_max={pre_cache_max}, post={post_cache_sizes:?}" ); } } // ──────────────────────────────────────────────────────────────────────────── // 4. Routing table recovers after partition heals (dead reprobe) // ──────────────────────────────────────────────────────────────────────────── #[test] fn routing_table_recovers_after_partition_heals() { // Given: 6-node cluster, partition at round 10, heal at round 40 // With dead_reprobe_interval=10, nodes re-discover dead members let config = DistributionSimConfig { name: "rt-recovery".into(), num_nodes: 6, num_rounds: 120, ticks_per_round: 3, actors_per_node: 0, swim: distribution::swim::probe::SwimConfig { probe_interval: 1, probe_timeout: 3, indirect_probes: 1, suspicion_timeout: 5, dead_reprobe_interval: 10, }, network_faults: vec![ NetworkFault::Partition { round: 10, partition: Partition { side_a: vec![0, 1, 2], side_b: vec![3, 4, 5], asymmetric: false, }, }, NetworkFault::Heal { round: 40 }, ], ..default_config() }; let (_trace, nodes) = run_simulation_with_nodes(config); // Then: after healing, all nodes should have recovered routing tables // Each node should see at least 4 of 5 other nodes in their routing table for (i, maybe_node) in nodes.iter().enumerate() { if let Some(node) = maybe_node { assert!( node.routing_table().len() >= 4, "node {i} should have ≥4 RT entries after partition heals, got {}", node.routing_table().len() ); } } } // ──────────────────────────────────────────────────────────────────────────── // 5. Routing table tracks alive membership (bounded invariant) // ──────────────────────────────────────────────────────────────────────────── #[test] fn routing_table_bounded_by_alive_count() { // Run a simulation with deaths and verify the routing table invariant let config = DistributionSimConfig { name: "rt-bounded".into(), num_nodes: 8, num_rounds: 80, ticks_per_round: 3, actors_per_node: 1, kill_schedule: vec![(15, 2), (25, 5)], ..default_config() }; let (trace, _) = run_simulation_with_nodes(config); let result = check_routing_table_bounded(&trace); assert!( result.passed, "routing table should never exceed alive count: {}", result.actual ); } // ──────────────────────────────────────────────────────────────────────────── // 6. Combined: partition + death during partition + heal + verify // ──────────────────────────────────────────────────────────────────────────── #[test] fn partition_then_death_during_partition_then_heal() { // Given: 6 nodes, partition at r=10, node 2 (side A) killed during partition // at r=20, heal at r=40. Node 2 had actors and a registry name. // Use high suspicion_timeout so cross-partition nodes stay Suspect (not Dead), // while within-partition death of node 2 is detected after timeout expires. let config = DistributionSimConfig { name: "partition-death-heal".into(), num_nodes: 6, num_rounds: 150, ticks_per_round: 3, actors_per_node: 2, network_faults: vec![ NetworkFault::Partition { round: 10, partition: Partition { side_a: vec![0, 1, 2], side_b: vec![3, 4, 5], asymmetric: false, }, }, NetworkFault::Heal { round: 40 }, ], action_schedule: vec![ (5, SimAction::RegisterName { node_idx: 2, name: "doomed-svc".into() }), (5, SimAction::RegisterName { node_idx: 4, name: "stable-svc".into() }), ], kill_schedule: vec![(20, 2)], swim: distribution::swim::probe::SwimConfig { probe_interval: 1, probe_timeout: 3, indirect_probes: 1, // >90 ticks (30 rounds × 3 ticks) so cross-partition nodes stay Suspect during // the 30-round partition. Node 2 (truly dead) gets declared dead ~33 rounds // after kill, well after partition heals. suspicion_timeout: 100, dead_reprobe_interval: 10, }, ..default_config() }; let (_trace, nodes) = run_simulation_with_nodes(config); let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); assert_eq!(survivors.len(), 5, "5 of 6 should survive"); // "doomed-svc" should be tombstoned (owner node 2 died) for node in &survivors { assert!( node.resolve_name("doomed-svc").is_none(), "doomed-svc should be tombstoned after owner died during partition" ); } // "stable-svc" should still resolve (node 4 alive throughout) for node in &survivors { assert!( node.resolve_name("stable-svc").is_some(), "stable-svc should resolve (owner survived partition)" ); } // After partition heal + dead reprobe, routing tables should recover // (at least 4 entries for each surviving node) for node in &survivors { assert!( node.routing_table().len() >= 3, "surviving node should have ≥3 RT entries after partition heals, got {}", node.routing_table().len() ); } } // ──────────────────────────────────────────────────────────────────────────── // 7. Registry GC: tombstones are garbage-collected after TTL // ──────────────────────────────────────────────────────────────────────────── #[test] fn registry_tombstones_gc_after_ttl() { // Given: short tombstone TTL and GC interval, register then unregister a name. // Then do many more register/unregister operations to advance the logical clock // (which is used for TTL comparison). After enough clock advancement, the // original tombstone should be garbage-collected. let config = DistributionSimConfig { name: "registry-gc".into(), num_nodes: 5, num_rounds: 80, ticks_per_round: 3, actors_per_node: 0, // Very short GC: TTL=5 logical clock ticks, GC runs every 3 ticks registry_tombstone_ttl: Some(5), registry_gc_interval: Some(3), action_schedule: vec![ (5, SimAction::RegisterName { node_idx: 0, name: "ephemeral".into() }), (10, SimAction::UnregisterName { node_idx: 0, name: "ephemeral".into() }), // Additional operations to advance the logical clock past the TTL (15, SimAction::RegisterName { node_idx: 1, name: "churn-1".into() }), (16, SimAction::RegisterName { node_idx: 2, name: "churn-2".into() }), (17, SimAction::RegisterName { node_idx: 3, name: "churn-3".into() }), (18, SimAction::RegisterName { node_idx: 4, name: "churn-4".into() }), (19, SimAction::RegisterName { node_idx: 1, name: "churn-5".into() }), (20, SimAction::RegisterName { node_idx: 2, name: "churn-6".into() }), ], ..default_config() }; let (trace, nodes) = run_simulation_with_nodes(config); // Shortly after unregister (round 12), tombstones should exist let mid_tombstones: usize = trace.snapshots_per_round .get(11) // round 12 .map(|snaps| { snaps.iter() .filter(|(_, s)| s.is_alive) .map(|(_, s)| s.registry_tombstone_count) .sum() }) .unwrap_or(0); assert!( mid_tombstones > 0, "tombstones should exist shortly after unregister" ); // After many more register operations advance the clock, the "ephemeral" tombstone // should be GC'd (its age exceeds TTL=5 in logical clock terms) let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); // Check that "ephemeral" resolves to None on all nodes (whether GC'd or still tombstoned) for node in &alive_nodes { assert!( node.resolve_name("ephemeral").is_none(), "ephemeral should not resolve (tombstoned or GC'd)" ); } // At least some nodes should have GC'd the tombstone (clock advanced past TTL) let nodes_with_ephemeral_tombstone: usize = alive_nodes .iter() .filter(|n| { n.registry().entries().any(|e| e.name == "ephemeral" && e.tombstone) }) .count(); assert!( nodes_with_ephemeral_tombstone < alive_nodes.len(), "at least some nodes should have GC'd the 'ephemeral' tombstone, but {} of {} still have it", nodes_with_ephemeral_tombstone, alive_nodes.len() ); }