feat: cycle 3 edge case registry tests — message loss, multi-kill, churn
New tests: - registry_converges_despite_message_loss (30% loss window + clean convergence) - simultaneous_kill_of_multiple_name_owners (3 name-owning nodes killed at once) - suspected_name_owner_recovers_and_registry_survives (Suspect→Alive path) - registry_correct_under_rapid_churn (interleaved kills + registrations + takeover) Total: 15 registry tests, 5 lifecycle, 5 property = 25 distribution sim tests. Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
This commit is contained in:
parent
9327d0734c
commit
57d9135c54
1 changed files with 217 additions and 1 deletions
|
|
@ -7,7 +7,7 @@ use simulation::distribution::properties::{
|
||||||
check_registry_propagation, check_registry_tombstones,
|
check_registry_propagation, check_registry_tombstones,
|
||||||
};
|
};
|
||||||
use simulation::distribution::sim::{
|
use simulation::distribution::sim::{
|
||||||
run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition,
|
run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition,
|
||||||
SimAction,
|
SimAction,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -506,3 +506,219 @@ fn piggyback_contention_both_propagate() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 12. Registry convergence under message loss
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registry_converges_despite_message_loss() {
|
||||||
|
// Given: 5-node cluster with a loss window during name registration,
|
||||||
|
// then clean connectivity for convergence.
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "registry-under-loss".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 100,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
registry_dissemination_lambda: Some(5),
|
||||||
|
network_faults: vec![
|
||||||
|
// Loss window: 30% drops during registration phase
|
||||||
|
NetworkFault::SetDropRate { round: 3, rate: 0.30 },
|
||||||
|
// Restore clean connectivity, forcing convergence via Alive transitions
|
||||||
|
NetworkFault::SetDropRate { round: 30, rate: 0.0 },
|
||||||
|
],
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 2, name: "beta".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 4, name: "gamma".into() }),
|
||||||
|
],
|
||||||
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
|
probe_interval: 1,
|
||||||
|
probe_timeout: 3,
|
||||||
|
indirect_probes: 3,
|
||||||
|
// High timeout prevents false deaths during the loss window.
|
||||||
|
suspicion_timeout: 200,
|
||||||
|
dead_reprobe_interval: 15,
|
||||||
|
},
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all surviving nodes should resolve all 3 names despite packet loss
|
||||||
|
let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||||
|
for node in &alive_nodes {
|
||||||
|
for name in &["alpha", "beta", "gamma"] {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name(name).is_some(),
|
||||||
|
"all nodes should resolve '{}' despite 20% message loss",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// All should agree on the same actor for each name
|
||||||
|
for name in &["alpha", "beta", "gamma"] {
|
||||||
|
let first = alive_nodes[0].resolve_name(name).unwrap().0;
|
||||||
|
assert!(
|
||||||
|
alive_nodes.iter().all(|n| n.resolve_name(name).unwrap().0 == first),
|
||||||
|
"all nodes should agree on actor for '{}' despite message loss",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 13. Multiple name-owning nodes killed simultaneously
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn simultaneous_kill_of_multiple_name_owners() {
|
||||||
|
// Given: 7-node cluster, nodes 0-2 each own a name, all 3 killed at round 15
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "multi-owner-kill".into(),
|
||||||
|
num_nodes: 7,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-a".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 1, name: "svc-b".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 2, name: "svc-c".into() }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(15, 0), (15, 1), (15, 2)],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all 4 survivors should resolve all 3 names to None (tombstoned)
|
||||||
|
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||||
|
assert_eq!(survivors.len(), 4, "4 of 7 nodes should survive");
|
||||||
|
|
||||||
|
for node in &survivors {
|
||||||
|
for name in &["svc-a", "svc-b", "svc-c"] {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name(name).is_none(),
|
||||||
|
"survivor should resolve '{}' to None after owner died",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 14. Name owner suspected but recovers — registry entry survives
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn suspected_name_owner_recovers_and_registry_survives() {
|
||||||
|
// Given: 5-node cluster, node 0 registers "svc",
|
||||||
|
// then a brief partition isolates node 0 (becomes Suspect, recovers before Dead)
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "suspect-recovery-registry".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
],
|
||||||
|
// Brief partition: isolate node 0 for 10 rounds (not long enough to reach Dead)
|
||||||
|
network_faults: vec![
|
||||||
|
NetworkFault::Partition {
|
||||||
|
round: 15,
|
||||||
|
partition: Partition {
|
||||||
|
side_a: vec![0],
|
||||||
|
side_b: vec![1, 2, 3, 4],
|
||||||
|
asymmetric: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
NetworkFault::Heal { round: 25 },
|
||||||
|
],
|
||||||
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
|
probe_interval: 1,
|
||||||
|
probe_timeout: 3,
|
||||||
|
indirect_probes: 1,
|
||||||
|
// Node stays Suspect during the 10-round partition (30 ticks < 200)
|
||||||
|
suspicion_timeout: 200,
|
||||||
|
dead_reprobe_interval: 10,
|
||||||
|
},
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all nodes should still resolve "svc" (node 0 never died, only suspected)
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("svc"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_some()),
|
||||||
|
"all nodes should resolve 'svc' after owner recovers from Suspect, got: {resolutions:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 15. Rapid churn: interleaved kills and registrations
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registry_correct_under_rapid_churn() {
|
||||||
|
// Given: 8-node cluster with interleaved kills and registrations
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "rapid-churn-registry".into(),
|
||||||
|
num_nodes: 8,
|
||||||
|
num_rounds: 100,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc-0".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 1, name: "svc-1".into() }),
|
||||||
|
(8, SimAction::RegisterName { node_idx: 4, name: "svc-4".into() }),
|
||||||
|
// Node 3 registers AFTER nodes 0 and 1 are killed
|
||||||
|
(20, SimAction::RegisterName { node_idx: 3, name: "svc-3".into() }),
|
||||||
|
// Node 5 takes over "svc-0" after original owner dies
|
||||||
|
(30, SimAction::RegisterName { node_idx: 5, name: "svc-0".into() }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(10, 0), (10, 1), (25, 2)],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
|
||||||
|
assert_eq!(survivors.len(), 5, "5 of 8 nodes should survive");
|
||||||
|
|
||||||
|
// svc-0 should resolve to node 5's re-registration (not tombstoned)
|
||||||
|
for node in &survivors {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc-0").is_some(),
|
||||||
|
"svc-0 should resolve (re-registered by node 5 after owner death)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// svc-1 should be tombstoned (node 1 died, no re-registration)
|
||||||
|
for node in &survivors {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc-1").is_none(),
|
||||||
|
"svc-1 should be tombstoned (owner died, not re-registered)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// svc-3 should resolve (registered after kills, node 3 alive)
|
||||||
|
for node in &survivors {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc-3").is_some(),
|
||||||
|
"svc-3 should resolve (registered after kills)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// svc-4 should resolve (node 4 alive throughout)
|
||||||
|
for node in &survivors {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc-4").is_some(),
|
||||||
|
"svc-4 should resolve (owner alive throughout)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue