diff --git a/crates/simulation/tests/distribution_registry.rs b/crates/simulation/tests/distribution_registry.rs index d312266..5726245 100644 --- a/crates/simulation/tests/distribution_registry.rs +++ b/crates/simulation/tests/distribution_registry.rs @@ -7,7 +7,7 @@ use simulation::distribution::properties::{ check_registry_propagation, check_registry_tombstones, }; use simulation::distribution::sim::{ - run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, + run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition, SimAction, }; @@ -506,3 +506,219 @@ fn piggyback_contention_both_propagate() { } } } + +// ──────────────────────────────────────────────────────────────────────────── +// 12. Registry convergence under message loss +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn registry_converges_despite_message_loss() { + // Given: 5-node cluster with a loss window during name registration, + // then clean connectivity for convergence. + let config = DistributionSimConfig { + name: "registry-under-loss".into(), + num_nodes: 5, + num_rounds: 100, + ticks_per_round: 3, + registry_dissemination_lambda: Some(5), + network_faults: vec![ + // Loss window: 30% drops during registration phase + NetworkFault::SetDropRate { round: 3, rate: 0.30 }, + // Restore clean connectivity, forcing convergence via Alive transitions + NetworkFault::SetDropRate { round: 30, rate: 0.0 }, + ], + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }), + (5, SimAction::RegisterName { node_idx: 2, name: "beta".into() }), + (5, SimAction::RegisterName { node_idx: 4, name: "gamma".into() }), + ], + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 3, + // High timeout prevents false deaths during the loss window. + suspicion_timeout: 200, + dead_reprobe_interval: 15, + }, + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // Then: all surviving nodes should resolve all 3 names despite packet loss + let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); + for node in &alive_nodes { + for name in &["alpha", "beta", "gamma"] { + assert!( + node.resolve_name(name).is_some(), + "all nodes should resolve '{}' despite 20% message loss", + name + ); + } + } + + // All should agree on the same actor for each name + for name in &["alpha", "beta", "gamma"] { + let first = alive_nodes[0].resolve_name(name).unwrap().0; + assert!( + alive_nodes.iter().all(|n| n.resolve_name(name).unwrap().0 == first), + "all nodes should agree on actor for '{}' despite message loss", + name + ); + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 13. Multiple name-owning nodes killed simultaneously +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn simultaneous_kill_of_multiple_name_owners() { + // Given: 7-node cluster, nodes 0-2 each own a name, all 3 killed at round 15 + let config = DistributionSimConfig { + name: "multi-owner-kill".into(), + num_nodes: 7, + num_rounds: 80, + ticks_per_round: 3, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "svc-a".into() }), + (5, SimAction::RegisterName { node_idx: 1, name: "svc-b".into() }), + (5, SimAction::RegisterName { node_idx: 2, name: "svc-c".into() }), + ], + kill_schedule: vec![(15, 0), (15, 1), (15, 2)], + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // Then: all 4 survivors should resolve all 3 names to None (tombstoned) + let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); + assert_eq!(survivors.len(), 4, "4 of 7 nodes should survive"); + + for node in &survivors { + for name in &["svc-a", "svc-b", "svc-c"] { + assert!( + node.resolve_name(name).is_none(), + "survivor should resolve '{}' to None after owner died", + name + ); + } + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// 14. Name owner suspected but recovers — registry entry survives +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn suspected_name_owner_recovers_and_registry_survives() { + // Given: 5-node cluster, node 0 registers "svc", + // then a brief partition isolates node 0 (becomes Suspect, recovers before Dead) + let config = DistributionSimConfig { + name: "suspect-recovery-registry".into(), + num_nodes: 5, + num_rounds: 80, + ticks_per_round: 3, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }), + ], + // Brief partition: isolate node 0 for 10 rounds (not long enough to reach Dead) + network_faults: vec![ + NetworkFault::Partition { + round: 15, + partition: Partition { + side_a: vec![0], + side_b: vec![1, 2, 3, 4], + asymmetric: false, + }, + }, + NetworkFault::Heal { round: 25 }, + ], + swim: distribution::swim::probe::SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 1, + // Node stays Suspect during the 10-round partition (30 ticks < 200) + suspicion_timeout: 200, + dead_reprobe_interval: 10, + }, + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + // Then: all nodes should still resolve "svc" (node 0 never died, only suspected) + let resolutions: Vec<_> = nodes + .iter() + .filter_map(|n| n.as_ref()) + .map(|n| n.resolve_name("svc")) + .collect(); + + assert!( + resolutions.iter().all(|r| r.is_some()), + "all nodes should resolve 'svc' after owner recovers from Suspect, got: {resolutions:?}" + ); +} + +// ──────────────────────────────────────────────────────────────────────────── +// 15. Rapid churn: interleaved kills and registrations +// ──────────────────────────────────────────────────────────────────────────── + +#[test] +fn registry_correct_under_rapid_churn() { + // Given: 8-node cluster with interleaved kills and registrations + let config = DistributionSimConfig { + name: "rapid-churn-registry".into(), + num_nodes: 8, + num_rounds: 100, + ticks_per_round: 3, + action_schedule: vec![ + (5, SimAction::RegisterName { node_idx: 0, name: "svc-0".into() }), + (5, SimAction::RegisterName { node_idx: 1, name: "svc-1".into() }), + (8, SimAction::RegisterName { node_idx: 4, name: "svc-4".into() }), + // Node 3 registers AFTER nodes 0 and 1 are killed + (20, SimAction::RegisterName { node_idx: 3, name: "svc-3".into() }), + // Node 5 takes over "svc-0" after original owner dies + (30, SimAction::RegisterName { node_idx: 5, name: "svc-0".into() }), + ], + kill_schedule: vec![(10, 0), (10, 1), (25, 2)], + ..default_config() + }; + + let (_trace, nodes) = run_simulation_with_nodes(config); + + let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect(); + assert_eq!(survivors.len(), 5, "5 of 8 nodes should survive"); + + // svc-0 should resolve to node 5's re-registration (not tombstoned) + for node in &survivors { + assert!( + node.resolve_name("svc-0").is_some(), + "svc-0 should resolve (re-registered by node 5 after owner death)" + ); + } + + // svc-1 should be tombstoned (node 1 died, no re-registration) + for node in &survivors { + assert!( + node.resolve_name("svc-1").is_none(), + "svc-1 should be tombstoned (owner died, not re-registered)" + ); + } + + // svc-3 should resolve (registered after kills, node 3 alive) + for node in &survivors { + assert!( + node.resolve_name("svc-3").is_some(), + "svc-3 should resolve (registered after kills)" + ); + } + + // svc-4 should resolve (node 4 alive throughout) + for node in &survivors { + assert!( + node.resolve_name("svc-4").is_some(), + "svc-4 should resolve (owner alive throughout)" + ); + } +}