feat: edge case registry sim tests + GracefulLeave action
Cycle 2 tests: - explicit_unregister_propagates_to_all_nodes - re_registration_after_tombstone_succeeds - all_names_tombstoned_when_owner_dies - graceful_leave_tombstones_registry_names - piggyback_contention_both_propagate Also: - Add GracefulLeave variant to SimAction enum - Fix cluster_survives_brief_message_loss config (dead_reprobe + suspicion_timeout) - Harden split_brain test config (suspicion_timeout=200 prevents cascading false deaths) Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
This commit is contained in:
parent
1c07f02be2
commit
9327d0734c
3 changed files with 238 additions and 5 deletions
|
|
@ -30,6 +30,8 @@ pub enum SimAction {
|
||||||
RegisterNameWithActor { node_idx: usize, name: String, actor: ActorAddress },
|
RegisterNameWithActor { node_idx: usize, name: String, actor: ActorAddress },
|
||||||
/// Unregister a name on the given node (creates a tombstone).
|
/// Unregister a name on the given node (creates a tombstone).
|
||||||
UnregisterName { node_idx: usize, name: String },
|
UnregisterName { node_idx: usize, name: String },
|
||||||
|
/// Graceful leave — node announces its own death before being removed.
|
||||||
|
GracefulLeave { node_idx: usize },
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Schedule entry for network faults.
|
/// Schedule entry for network faults.
|
||||||
|
|
@ -415,6 +417,43 @@ fn run_simulation_inner(config: DistributionSimConfig) -> (DistTrace, Vec<Option
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
SimAction::GracefulLeave { node_idx } => {
|
||||||
|
if *node_idx < n {
|
||||||
|
if let Some(ref mut node) = nodes[*node_idx] {
|
||||||
|
let leave_actions = node.leave();
|
||||||
|
// Deliver the leave actions (disseminate death announcement)
|
||||||
|
let tagged_responses = deliver_actions_tagged_with_net(
|
||||||
|
&leave_actions,
|
||||||
|
*node_idx,
|
||||||
|
node_ids[*node_idx],
|
||||||
|
addrs[*node_idx],
|
||||||
|
&mut nodes,
|
||||||
|
&node_ids,
|
||||||
|
&addrs,
|
||||||
|
&mut net,
|
||||||
|
);
|
||||||
|
for (responder_idx, response_actions) in tagged_responses {
|
||||||
|
deliver_actions_tagged_with_net(
|
||||||
|
&response_actions,
|
||||||
|
responder_idx,
|
||||||
|
node_ids[responder_idx],
|
||||||
|
addrs[responder_idx],
|
||||||
|
&mut nodes,
|
||||||
|
&node_ids,
|
||||||
|
&addrs,
|
||||||
|
&mut net,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Remove the node after leave
|
||||||
|
nodes[*node_idx] = None;
|
||||||
|
events.push(Event {
|
||||||
|
tick: round as u64,
|
||||||
|
node_name: node_names[*node_idx].clone(),
|
||||||
|
kind: DistributionEventKind::NodeKilled,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -630,7 +630,9 @@ fn sequential_partitions_fragment_cluster() {
|
||||||
#[test]
|
#[test]
|
||||||
fn cluster_survives_brief_message_loss() {
|
fn cluster_survives_brief_message_loss() {
|
||||||
// Given: 5 nodes with 15% loss for a brief window, then clean network.
|
// Given: 5 nodes with 15% loss for a brief window, then clean network.
|
||||||
// High suspicion timeout prevents false positives during the loss period.
|
// High suspicion timeout + dead reprobe prevents permanent false positives.
|
||||||
|
// Without dead reprobe, correct death dissemination causes cascading
|
||||||
|
// false deaths that collapse the cluster.
|
||||||
let config = DistributionSimConfig {
|
let config = DistributionSimConfig {
|
||||||
name: "brief-loss-recovery".into(),
|
name: "brief-loss-recovery".into(),
|
||||||
num_nodes: 5,
|
num_nodes: 5,
|
||||||
|
|
@ -640,9 +642,9 @@ fn cluster_survives_brief_message_loss() {
|
||||||
swim: distribution::swim::probe::SwimConfig {
|
swim: distribution::swim::probe::SwimConfig {
|
||||||
probe_interval: 1,
|
probe_interval: 1,
|
||||||
probe_timeout: 5,
|
probe_timeout: 5,
|
||||||
indirect_probes: 2,
|
indirect_probes: 3,
|
||||||
suspicion_timeout: 20,
|
suspicion_timeout: 60,
|
||||||
dead_reprobe_interval: 0,
|
dead_reprobe_interval: 15,
|
||||||
},
|
},
|
||||||
network_faults: vec![
|
network_faults: vec![
|
||||||
NetworkFault::SetDropRate {
|
NetworkFault::SetDropRate {
|
||||||
|
|
|
||||||
|
|
@ -100,7 +100,10 @@ fn split_brain_naming_converges_after_partition_heals() {
|
||||||
probe_interval: 1,
|
probe_interval: 1,
|
||||||
probe_timeout: 3,
|
probe_timeout: 3,
|
||||||
indirect_probes: 1,
|
indirect_probes: 1,
|
||||||
suspicion_timeout: 5,
|
// High enough that no node reaches Dead during the 30-round partition.
|
||||||
|
// Nodes go Suspect → back to Alive when partition heals, triggering
|
||||||
|
// re_disseminate_all which propagates both sides' registry entries.
|
||||||
|
suspicion_timeout: 200,
|
||||||
dead_reprobe_interval: 10,
|
dead_reprobe_interval: 10,
|
||||||
},
|
},
|
||||||
..default_config()
|
..default_config()
|
||||||
|
|
@ -314,3 +317,192 @@ fn multiple_names_from_different_nodes_all_propagate() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 7. Explicit unregister propagates to all nodes
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_unregister_propagates_to_all_nodes() {
|
||||||
|
// Given: 5-node cluster, node 0 registers "svc" at round 5, unregisters at round 15
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "explicit-unregister".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 60,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
(15, SimAction::UnregisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all nodes should resolve "svc" to None (tombstoned)
|
||||||
|
for (i, node) in nodes.iter().filter_map(|n| n.as_ref()).enumerate() {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name("svc").is_none(),
|
||||||
|
"node {i} should resolve 'svc' to None after unregister"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 8. Re-registration after tombstone overwrites the tombstone
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn re_registration_after_tombstone_succeeds() {
|
||||||
|
// Given: node 0 registers "svc", then it's killed (tombstoned),
|
||||||
|
// then node 1 re-registers "svc" with a new actor
|
||||||
|
use swactor::actor::ActorAddress;
|
||||||
|
let new_actor = ActorAddress::new_random();
|
||||||
|
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "re-register-after-tombstone".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 100,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
// Node 1 re-registers "svc" well after node 0 dies and tombstone propagates
|
||||||
|
(50, SimAction::RegisterNameWithActor { node_idx: 1, name: "svc".into(), actor: new_actor }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(15, 0)],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all survivors should resolve "svc" to the new actor from node 1
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("svc"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_some()),
|
||||||
|
"all survivors should resolve 'svc' to the new registration, got: {resolutions:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.unwrap().0 == new_actor),
|
||||||
|
"all survivors should resolve 'svc' to the new actor"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 9. Multiple names from same node, kill node, all tombstoned
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_names_tombstoned_when_owner_dies() {
|
||||||
|
// Given: node 0 registers 3 names, then is killed
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "multi-name-tombstone".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "beta".into() }),
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "gamma".into() }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(15, 0)],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all survivors should resolve all 3 names to None
|
||||||
|
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
||||||
|
for name in &["alpha", "beta", "gamma"] {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name(name).is_none(),
|
||||||
|
"all names should be tombstoned after owner dies, but '{}' still resolves",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 10. Graceful leave tombstones the leaving node's names
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn graceful_leave_tombstones_registry_names() {
|
||||||
|
// Given: node 0 registers "svc", then does a graceful leave
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "graceful-leave-registry".into(),
|
||||||
|
num_nodes: 5,
|
||||||
|
num_rounds: 80,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
action_schedule: vec![
|
||||||
|
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
|
||||||
|
(20, SimAction::GracefulLeave { node_idx: 0 }),
|
||||||
|
],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all survivors should resolve "svc" to None (tombstoned via death notification)
|
||||||
|
let resolutions: Vec<_> = nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| n.as_ref())
|
||||||
|
.map(|n| n.resolve_name("svc"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
resolutions.iter().all(|r| r.is_none()),
|
||||||
|
"all survivors should resolve 'svc' to None after graceful leave, got: {resolutions:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// 11. Piggyback contention — kills + registrations compete for bandwidth
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn piggyback_contention_both_propagate() {
|
||||||
|
// Given: 10-node cluster, kill 2 nodes + register 3 names simultaneously
|
||||||
|
// Both membership death updates and registry entries share piggyback bandwidth
|
||||||
|
let config = DistributionSimConfig {
|
||||||
|
name: "piggyback-contention".into(),
|
||||||
|
num_nodes: 10,
|
||||||
|
num_rounds: 100,
|
||||||
|
ticks_per_round: 3,
|
||||||
|
actors_per_node: 0,
|
||||||
|
action_schedule: vec![
|
||||||
|
(10, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
|
||||||
|
(10, SimAction::RegisterName { node_idx: 3, name: "beta".into() }),
|
||||||
|
(10, SimAction::RegisterName { node_idx: 6, name: "gamma".into() }),
|
||||||
|
],
|
||||||
|
kill_schedule: vec![(10, 2), (10, 5)],
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
|
||||||
|
let (trace, nodes) = run_simulation_with_nodes(config);
|
||||||
|
|
||||||
|
// Then: all survivors should have all 3 registry names
|
||||||
|
let result = check_registry_propagation(&trace, 3);
|
||||||
|
assert!(
|
||||||
|
result.passed,
|
||||||
|
"all 3 names should propagate despite contention with death updates: {}",
|
||||||
|
result.actual
|
||||||
|
);
|
||||||
|
|
||||||
|
// And: all survivors should resolve all 3 names
|
||||||
|
for node in nodes.iter().filter_map(|n| n.as_ref()) {
|
||||||
|
for name in &["alpha", "beta", "gamma"] {
|
||||||
|
assert!(
|
||||||
|
node.resolve_name(name).is_some(),
|
||||||
|
"survivor should resolve '{}' despite piggyback contention",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue