zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: lifecycle and property sim tests, remove debug test

Browse source 
Phase 3 — lifecycle sim tests (distribution_lifecycle.rs):
- dead_node_triggers_repair_queue_and_cache_invalidation
- revived_node_has_empty_directory
- cache_shrinks_after_node_death
- routing_table_recovers_after_partition_heals
- routing_table_bounded_by_alive_count

Phase 4 — property-based sim tests (distribution_properties.rs):
- routing_table_bounded_across_configs (3 config variants)
- cache_bounded_across_configs (2 config variants)
- repair_queue_populates_on_death_with_directory_entries
- registry_eventually_consistent_across_configs (3 config variants)
- cascading_deaths_maintain_invariants

Also: update cluster_scenarios 10% message loss test to use
suspicion_timeout=60 + indirect_probes=3 + dead_reprobe=15
for resilience under correct death dissemination.

Remove diagnostic debug_registry.rs (superseded by distribution_registry.rs).

Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
This commit is contained in:
Claude 2026-02-13 08:53:07 +00:00
parent 5d8e413db2
commit 3226ba3764
4 changed files with 500 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
crates/simulation/tests/cluster_scenarios.rs
View file

@ -138,9 +138,9 @@ fn cluster_converges_under_10_percent_message_loss() {
swim: distribution::swim::probe::SwimConfig { swim: distribution::swim::probe::SwimConfig {
probe_interval: 1, probe_interval: 1,
probe_timeout: 5, probe_timeout: 5,
indirect_probes: 2, indirect_probes: 3,
suspicion_timeout: 20, suspicion_timeout: 60,
dead_reprobe_interval: 30, dead_reprobe_interval: 15,
}, },
network_faults: vec![NetworkFault::SetDropRate { network_faults: vec![NetworkFault::SetDropRate {
round: 1, round: 1,

56
crates/simulation/tests/debug_registry.rs
View file

@ -1,56 +0,0 @@
use simulation::distribution::sim::{
run_simulation_with_nodes, DistributionSimConfig, SimAction,
};
#[test]
fn debug_tombstone_propagation() {
let config = DistributionSimConfig {
name: "debug-tombstone".into(),
num_nodes: 5,
num_rounds: 80,
ticks_per_round: 3,
actors_per_node: 0,
action_schedule: vec![
(5, SimAction::RegisterName { node_idx: 0, name: "svc".into() }),
],
kill_schedule: vec![(15, 0)],
..DistributionSimConfig::default()
};
let (trace, nodes) = run_simulation_with_nodes(config);
// Check snapshots at various rounds
for round_idx in [4, 9, 14, 19, 24, 39, 59, 79] {
if round_idx < trace.snapshots_per_round.len() {
let snaps = &trace.snapshots_per_round[round_idx];
let reg_sizes: Vec<usize> = snaps.iter().map(|(_, s)| s.registry_size).collect();
let tomb_counts: Vec<usize> = snaps.iter().map(|(_, s)| s.registry_tombstone_count).collect();
let alive: Vec<bool> = snaps.iter().map(|(_, s)| s.is_alive).collect();
let members: Vec<usize> = snaps.iter().map(|(_, s)| s.member_count).collect();
eprintln!("Round {}: alive={alive:?} members={members:?} registry={reg_sizes:?} tombstones={tomb_counts:?}", round_idx + 1);
}
}
// Check membership change events
let dead_events: Vec<_> = trace.events.iter()
.filter(|e| matches!(&e.kind, simulation::distribution::trace::DistributionEventKind::MembershipChanged { new_state, .. } if new_state == "Dead"))
.collect();
eprintln!("Dead events: {}", dead_events.len());
for e in &dead_events {
if let simulation::distribution::trace::DistributionEventKind::MembershipChanged { target, new_state } = &e.kind {
eprintln!(" tick={} node={} declared {target} as {new_state}", e.tick, e.node_name);
}
}
// Check final nodes' resolve_name
for (i, node) in nodes.iter().enumerate() {
match node {
Some(n) => {
let res = n.resolve_name("svc");
eprintln!("Node {i}: resolve_name('svc') = {res:?}, registry_size={}, tombstones={}",
n.registry().len(), n.registry().tombstone_count());
}
None => eprintln!("Node {i}: DEAD"),
}
}
}

237
crates/simulation/tests/distribution_lifecycle.rs Normal file
View file

@ -0,0 +1,237 @@
//! Lifecycle simulation tests — death/repair/cache/routing behavior.
//!
//! Tests that node death correctly triggers:
//! - Repair queue population for re-replication
//! - Cache invalidation of stale entries
//! - Routing table cleanup
//! - Recovery after partition heals
use simulation::distribution::properties::{
check_repair_queue_populated, check_routing_table_bounded,
};
use simulation::distribution::sim::{
run_simulation, run_simulation_with_nodes, DistributionSimConfig, NetworkFault, Partition,
};
fn default_config() -> DistributionSimConfig {
DistributionSimConfig::default()
}
// ────────────────────────────────────────────────────────────────────────────
// 1. Dead node's actors populate repair queue and invalidate cache
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn dead_node_triggers_repair_queue_and_cache_invalidation() {
// Given: 5-node cluster, 2 actors/node. Node 2 is killed at round 10.
let config = DistributionSimConfig {
name: "death-repair-cache".into(),
num_nodes: 5,
num_rounds: 80,
ticks_per_round: 3,
actors_per_node: 2,
kill_schedule: vec![(10, 2)],
..default_config()
};
let (trace, nodes) = run_simulation_with_nodes(config);
// Then: at least one survivor should have a non-empty repair queue
let result = check_repair_queue_populated(&trace, 10);
assert!(
result.passed,
"repair queue should be populated after node death: {}",
result.actual
);
// And: no survivor's cache should contain entries pointing to the dead node
let dead_node_id = {
// Find the node_id for node 2 from round snapshots before death
// We can check from the surviving nodes
// Node 2 is dead (None), so we check survivors' caches
let mut stale_count = 0;
for node in nodes.iter().filter_map(|n| n.as_ref()) {
for (_actor, cached_on) in node.cache().entries() {
// The dead node's entries should have been invalidated
// We can't easily get node 2's ID here, but we can check
// that no survivor caches an actor on a node not in their members
let alive_ids: Vec<_> = node.members().iter().map(|m| m.node_id).collect();
if !alive_ids.contains(&cached_on) && cached_on != node.node_id() {
stale_count += 1;
}
}
}
stale_count
};
assert_eq!(
dead_node_id, 0,
"no survivor should have cache entries pointing to non-member nodes"
);
}
// ────────────────────────────────────────────────────────────────────────────
// 2. Revived node starts fresh (empty directory)
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn revived_node_has_empty_directory() {
// Given: 5-node cluster, 2 actors/node.
// Node 2 killed at round 10, revived at round 50.
let config = DistributionSimConfig {
name: "revive-fresh".into(),
num_nodes: 5,
num_rounds: 100,
ticks_per_round: 3,
actors_per_node: 2,
kill_schedule: vec![(10, 2)],
revive_schedule: vec![(50, 2)],
..default_config()
};
let (_trace, nodes) = run_simulation_with_nodes(config);
// Then: the revived node should have an empty directory
// (it's a fresh DistributedNode, not carrying over old state)
let revived = nodes[2].as_ref().expect("node 2 should be revived");
assert_eq!(
revived.directory().entry_count(), 0,
"revived node should start with empty directory"
);
// And: the revived node should have rejoined the cluster
assert!(
!revived.members().is_empty(),
"revived node should have some cluster members"
);
}
// ────────────────────────────────────────────────────────────────────────────
// 3. Cache invalidation tracks membership changes
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn cache_shrinks_after_node_death() {
// Given: 5-node cluster with actors, all caches populated during setup.
// When: node 1 is killed
// Then: cache_size should decrease for survivors after death detection.
let config = DistributionSimConfig {
name: "cache-invalidation".into(),
num_nodes: 5,
num_rounds: 80,
ticks_per_round: 3,
actors_per_node: 3,
kill_schedule: vec![(10, 1)],
..default_config()
};
let (trace, _nodes) = run_simulation_with_nodes(config);
// Check that cache_size decreased for at least some survivors after death
// Before death (round 9), survivors should have cache entries
// After death detection, cache for dead node's actors should be invalidated
let pre_death_round = 8; // 0-indexed round 9
let post_detection_round = 39; // well after SWIM detection
if pre_death_round < trace.snapshots_per_round.len()
&& post_detection_round < trace.snapshots_per_round.len()
{
let pre_cache_max: usize = trace.snapshots_per_round[pre_death_round]
.iter()
.filter(|(_, s)| s.is_alive)
.map(|(_, s)| s.cache_size)
.max()
.unwrap_or(0);
let post_cache_sizes: Vec<usize> = trace.snapshots_per_round[post_detection_round]
.iter()
.filter(|(_, s)| s.is_alive)
.map(|(_, s)| s.cache_size)
.collect();
// After death, some survivors should have fewer cache entries
// (the dead node's actors were invalidated)
let any_decreased = post_cache_sizes.iter().any(|&s| s < pre_cache_max);
assert!(
any_decreased || pre_cache_max == 0,
"cache should shrink after node death, pre_max={pre_cache_max}, post={post_cache_sizes:?}"
);
}
}
// ────────────────────────────────────────────────────────────────────────────
// 4. Routing table recovers after partition heals (dead reprobe)
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn routing_table_recovers_after_partition_heals() {
// Given: 6-node cluster, partition at round 10, heal at round 40
// With dead_reprobe_interval=10, nodes re-discover dead members
let config = DistributionSimConfig {
name: "rt-recovery".into(),
num_nodes: 6,
num_rounds: 120,
ticks_per_round: 3,
actors_per_node: 0,
swim: distribution::swim::probe::SwimConfig {
probe_interval: 1,
probe_timeout: 3,
indirect_probes: 1,
suspicion_timeout: 5,
dead_reprobe_interval: 10,
},
network_faults: vec![
NetworkFault::Partition {
round: 10,
partition: Partition {
side_a: vec![0, 1, 2],
side_b: vec![3, 4, 5],
asymmetric: false,
},
},
NetworkFault::Heal { round: 40 },
],
..default_config()
};
let (_trace, nodes) = run_simulation_with_nodes(config);
// Then: after healing, all nodes should have recovered routing tables
// Each node should see at least 4 of 5 other nodes in their routing table
for (i, maybe_node) in nodes.iter().enumerate() {
if let Some(node) = maybe_node {
assert!(
node.routing_table().len() >= 4,
"node {i} should have ≥4 RT entries after partition heals, got {}",
node.routing_table().len()
);
}
}
}
// ────────────────────────────────────────────────────────────────────────────
// 5. Routing table tracks alive membership (bounded invariant)
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn routing_table_bounded_by_alive_count() {
// Run a simulation with deaths and verify the routing table invariant
let config = DistributionSimConfig {
name: "rt-bounded".into(),
num_nodes: 8,
num_rounds: 80,
ticks_per_round: 3,
actors_per_node: 1,
kill_schedule: vec![(15, 2), (25, 5)],
..default_config()
};
let (trace, _) = run_simulation_with_nodes(config);
let result = check_routing_table_bounded(&trace);
assert!(
result.passed,
"routing table should never exceed alive count: {}",
result.actual
);
}

260
crates/simulation/tests/distribution_properties.rs Normal file
View file

@ -0,0 +1,260 @@
//! Property-based distribution tests — invariants that must hold across configs.
//!
//! Each test verifies a structural property across multiple simulation
//! configurations with varying fault conditions.
use simulation::distribution::properties::{
check_cache_bounded, check_registry_propagation, check_repair_queue_populated,
check_routing_table_bounded,
};
use simulation::distribution::sim::{
run_simulation_with_nodes, DistributionSimConfig, SimAction,
};
// ────────────────────────────────────────────────────────────────────────────
// 1. Routing table size ≤ alive membership at every round
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn routing_table_bounded_across_configs() {
let configs = vec![
// Healthy 5-node cluster
DistributionSimConfig {
name: "rt-bound-healthy-5".into(),
num_nodes: 5,
num_rounds: 50,
ticks_per_round: 3,
..DistributionSimConfig::default()
},
// 10-node cluster with 2 deaths
DistributionSimConfig {
name: "rt-bound-deaths-10".into(),
num_nodes: 10,
num_rounds: 60,
ticks_per_round: 3,
kill_schedule: vec![(15, 3), (25, 7)],
..DistributionSimConfig::default()
},
// 15-node cluster with 1 death
DistributionSimConfig {
name: "rt-bound-large-15".into(),
num_nodes: 15,
num_rounds: 60,
ticks_per_round: 3,
kill_schedule: vec![(20, 0)],
..DistributionSimConfig::default()
},
];
for config in configs {
let name = config.name.clone();
let (trace, _) = run_simulation_with_nodes(config);
let result = check_routing_table_bounded(&trace);
assert!(
result.passed,
"[{name}] routing table bounded invariant violated: {}",
result.actual
);
}
}
// ────────────────────────────────────────────────────────────────────────────
// 2. Cache size ≤ cache_capacity at every round
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn cache_bounded_across_configs() {
let capacity = 100;
let configs = vec![
DistributionSimConfig {
name: "cache-bound-5".into(),
num_nodes: 5,
num_rounds: 50,
ticks_per_round: 3,
actors_per_node: 5,
cache_capacity: capacity,
..DistributionSimConfig::default()
},
DistributionSimConfig {
name: "cache-bound-10-deaths".into(),
num_nodes: 10,
num_rounds: 60,
ticks_per_round: 3,
actors_per_node: 3,
cache_capacity: capacity,
kill_schedule: vec![(15, 2), (20, 5)],
..DistributionSimConfig::default()
},
];
for config in configs {
let name = config.name.clone();
let (trace, _) = run_simulation_with_nodes(config);
let result = check_cache_bounded(&trace, capacity);
assert!(
result.passed,
"[{name}] cache bounded invariant violated: {}",
result.actual
);
}
}
// ────────────────────────────────────────────────────────────────────────────
// 3. Repair queue populates when node with directory entries dies
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn repair_queue_populates_on_death_with_directory_entries() {
// Node 2 has 3 actors. When it dies, repair queue should grow.
let config = DistributionSimConfig {
name: "repair-proportional".into(),
num_nodes: 5,
num_rounds: 80,
ticks_per_round: 3,
actors_per_node: 3,
kill_schedule: vec![(15, 2)],
..DistributionSimConfig::default()
};
let (trace, _) = run_simulation_with_nodes(config);
// Repair queue should be populated within 2-3 rounds of death detection
let result = check_repair_queue_populated(&trace, 15);
assert!(
result.passed,
"repair queue should fill after node with actors dies: {}",
result.actual
);
// Check that the total repair queue size across survivors is proportional
// to the dead node's directory entries (3 actors)
let post_death_sizes: Vec<usize> = trace
.snapshots_per_round
.iter()
.skip(20)
.take(30)
.flat_map(|round_snaps| {
round_snaps
.iter()
.filter(|(_, s)| s.is_alive)
.map(|(_, s)| s.repair_queue_size)
})
.collect();
let max_repair = post_death_sizes.iter().max().copied().unwrap_or(0);
assert!(
max_repair >= 1,
"at least one repair queue entry expected for dead node's actors, max seen: {max_repair}"
);
}
// ────────────────────────────────────────────────────────────────────────────
// 4. Registry convergence — eventual consistency across configs
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn registry_eventually_consistent_across_configs() {
let configs = vec![
// 5 nodes, 1 name
(
DistributionSimConfig {
name: "reg-ec-simple".into(),
num_nodes: 5,
num_rounds: 50,
ticks_per_round: 3,
actors_per_node: 0,
action_schedule: vec![
(5, SimAction::RegisterName { node_idx: 0, name: "svc-a".into() }),
],
..DistributionSimConfig::default()
},
1, // expected min registry size
),
// 8 nodes, 3 names from different nodes
(
DistributionSimConfig {
name: "reg-ec-multi".into(),
num_nodes: 8,
num_rounds: 60,
ticks_per_round: 3,
actors_per_node: 0,
action_schedule: vec![
(5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
(5, SimAction::RegisterName { node_idx: 3, name: "beta".into() }),
(5, SimAction::RegisterName { node_idx: 6, name: "gamma".into() }),
],
..DistributionSimConfig::default()
},
3,
),
// 5 nodes, register + kill owner, verify tombstone propagates
(
DistributionSimConfig {
name: "reg-ec-death".into(),
num_nodes: 5,
num_rounds: 80,
ticks_per_round: 3,
actors_per_node: 0,
action_schedule: vec![
(5, SimAction::RegisterName { node_idx: 0, name: "ephemeral".into() }),
],
kill_schedule: vec![(15, 0)],
..DistributionSimConfig::default()
},
1, // tombstoned entry still counts as registry_size
),
];
for (config, min_size) in configs {
let name = config.name.clone();
let (trace, _) = run_simulation_with_nodes(config);
let result = check_registry_propagation(&trace, min_size);
assert!(
result.passed,
"[{name}] registry should converge: {}",
result.actual
);
}
}
// ────────────────────────────────────────────────────────────────────────────
// 5. Multiple deaths don't violate invariants
// ────────────────────────────────────────────────────────────────────────────
#[test]
fn cascading_deaths_maintain_invariants() {
// 8 nodes, 3 die in sequence
let config = DistributionSimConfig {
name: "cascade-invariants".into(),
num_nodes: 8,
num_rounds: 100,
ticks_per_round: 3,
actors_per_node: 2,
cache_capacity: 200,
kill_schedule: vec![(10, 1), (20, 3), (30, 5)],
..DistributionSimConfig::default()
};
let (trace, _) = run_simulation_with_nodes(config);
let rt_result = check_routing_table_bounded(&trace);
assert!(
rt_result.passed,
"routing table bounded after cascading deaths: {}",
rt_result.actual
);
let cache_result = check_cache_bounded(&trace, 200);
assert!(
cache_result.passed,
"cache bounded after cascading deaths: {}",
cache_result.actual
);
let repair_result = check_repair_queue_populated(&trace, 10);
assert!(
repair_result.passed,
"repair queue populated after first death: {}",
repair_result.actual
);
}