swactor/crates/simulation/tests/distribution_properties.rs

#![cfg(feature = "distribution")]
//! Property-based distribution tests — invariants that must hold across configs.
//!
//! Each test verifies a structural property across multiple simulation
//! configurations with varying fault conditions.

use simulation::distribution::properties::{
    check_cache_bounded, check_registry_propagation, check_repair_queue_populated,
    check_routing_table_bounded,
};
use simulation::distribution::sim::{
    run_simulation_with_nodes, DistributionSimConfig, DistTrace, NetworkFault, Partition, SimAction,
};

fn maybe_save_trace(trace: &DistTrace) {
    if let Ok(dir) = std::env::var("SWACTOR_TRACE_DIR") {
        std::fs::create_dir_all(&dir).ok();
        let filename = format!(
            "{}/{}.trace.json",
            dir,
            trace.name.to_lowercase().replace(' ', "_").replace(['(', ')'], "")
        );
        let json = serde_json::to_string_pretty(trace).expect("trace serialization failed");
        std::fs::write(&filename, json).expect("trace write failed");
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 1. Routing table size ≤ alive membership at every round
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn routing_table_bounded_across_configs() {
    let configs = vec![
        // Healthy 5-node cluster
        DistributionSimConfig {
            name: "rt-bound-healthy-5".into(),
            num_nodes: 5,
            num_rounds: 50,
            ticks_per_round: 3,
            ..DistributionSimConfig::default()
        },
        // 10-node cluster with 2 deaths
        DistributionSimConfig {
            name: "rt-bound-deaths-10".into(),
            num_nodes: 10,
            num_rounds: 60,
            ticks_per_round: 3,
            kill_schedule: vec![(15, 3), (25, 7)],
            ..DistributionSimConfig::default()
        },
        // 15-node cluster with 1 death
        DistributionSimConfig {
            name: "rt-bound-large-15".into(),
            num_nodes: 15,
            num_rounds: 60,
            ticks_per_round: 3,
            kill_schedule: vec![(20, 0)],
            ..DistributionSimConfig::default()
        },
    ];

    for config in configs {
        let name = config.name.clone();
        let (trace, _) = run_simulation_with_nodes(config);
        maybe_save_trace(&trace);
        let result = check_routing_table_bounded(&trace);
        assert!(
            result.passed,
            "[{name}] routing table bounded invariant violated: {}",
            result.actual
        );
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 2. Cache size ≤ cache_capacity at every round
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cache_bounded_across_configs() {
    let capacity = 100;
    let configs = vec![
        DistributionSimConfig {
            name: "cache-bound-5".into(),
            num_nodes: 5,
            num_rounds: 50,
            ticks_per_round: 3,
            actors_per_node: 5,
            cache_capacity: capacity,
            ..DistributionSimConfig::default()
        },
        DistributionSimConfig {
            name: "cache-bound-10-deaths".into(),
            num_nodes: 10,
            num_rounds: 60,
            ticks_per_round: 3,
            actors_per_node: 3,
            cache_capacity: capacity,
            kill_schedule: vec![(15, 2), (20, 5)],
            ..DistributionSimConfig::default()
        },
    ];

    for config in configs {
        let name = config.name.clone();
        let (trace, _) = run_simulation_with_nodes(config);
        maybe_save_trace(&trace);
        let result = check_cache_bounded(&trace, capacity);
        assert!(
            result.passed,
            "[{name}] cache bounded invariant violated: {}",
            result.actual
        );
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 3. Repair queue populates when node with directory entries dies
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn repair_queue_populates_on_death_with_directory_entries() {
    // Node 2 has 3 actors. When it dies, repair queue should grow.
    let config = DistributionSimConfig {
        name: "repair-proportional".into(),
        num_nodes: 5,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 3,
        kill_schedule: vec![(15, 2)],
        ..DistributionSimConfig::default()
    };

    let (trace, _) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    // Repair queue should be populated within 2-3 rounds of death detection
    let result = check_repair_queue_populated(&trace, 15);
    assert!(
        result.passed,
        "repair queue should fill after node with actors dies: {}",
        result.actual
    );

    // Check that the total repair queue size across survivors is proportional
    // to the dead node's directory entries (3 actors)
    let post_death_sizes: Vec<usize> = trace
        .snapshots_per_round
        .iter()
        .skip(20)
        .take(30)
        .flat_map(|round_snaps| {
            round_snaps
                .iter()
                .filter(|(_, s)| s.is_alive)
                .map(|(_, s)| s.repair_queue_size)
        })
        .collect();

    let max_repair = post_death_sizes.iter().max().copied().unwrap_or(0);
    assert!(
        max_repair >= 1,
        "at least one repair queue entry expected for dead node's actors, max seen: {max_repair}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 4. Registry convergence — eventual consistency across configs
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn registry_eventually_consistent_across_configs() {
    let configs = vec![
        // 5 nodes, 1 name
        (
            DistributionSimConfig {
                name: "reg-ec-simple".into(),
                num_nodes: 5,
                num_rounds: 50,
                ticks_per_round: 3,
                actors_per_node: 0,
                action_schedule: vec![
                    (5, SimAction::RegisterName { node_idx: 0, name: "svc-a".into() }),
                ],
                ..DistributionSimConfig::default()
            },
            1, // expected min registry size
        ),
        // 8 nodes, 3 names from different nodes
        (
            DistributionSimConfig {
                name: "reg-ec-multi".into(),
                num_nodes: 8,
                num_rounds: 60,
                ticks_per_round: 3,
                actors_per_node: 0,
                action_schedule: vec![
                    (5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
                    (5, SimAction::RegisterName { node_idx: 3, name: "beta".into() }),
                    (5, SimAction::RegisterName { node_idx: 6, name: "gamma".into() }),
                ],
                ..DistributionSimConfig::default()
            },
            3,
        ),
        // 5 nodes, register + kill owner, verify tombstone propagates
        (
            DistributionSimConfig {
                name: "reg-ec-death".into(),
                num_nodes: 5,
                num_rounds: 80,
                ticks_per_round: 3,
                actors_per_node: 0,
                action_schedule: vec![
                    (5, SimAction::RegisterName { node_idx: 0, name: "ephemeral".into() }),
                ],
                kill_schedule: vec![(15, 0)],
                ..DistributionSimConfig::default()
            },
            1, // tombstoned entry still counts as registry_size
        ),
    ];

    for (config, min_size) in configs {
        let name = config.name.clone();
        let (trace, _) = run_simulation_with_nodes(config);
        maybe_save_trace(&trace);
        let result = check_registry_propagation(&trace, min_size);
        assert!(
            result.passed,
            "[{name}] registry should converge: {}",
            result.actual
        );
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 5. Multiple deaths don't violate invariants
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cascading_deaths_maintain_invariants() {
    // 8 nodes, 3 die in sequence
    let config = DistributionSimConfig {
        name: "cascade-invariants".into(),
        num_nodes: 8,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 2,
        cache_capacity: 200,
        kill_schedule: vec![(10, 1), (20, 3), (30, 5)],
        ..DistributionSimConfig::default()
    };

    let (trace, _) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    let rt_result = check_routing_table_bounded(&trace);
    assert!(
        rt_result.passed,
        "routing table bounded after cascading deaths: {}",
        rt_result.actual
    );

    let cache_result = check_cache_bounded(&trace, 200);
    assert!(
        cache_result.passed,
        "cache bounded after cascading deaths: {}",
        cache_result.actual
    );

    let repair_result = check_repair_queue_populated(&trace, 10);
    assert!(
        repair_result.passed,
        "repair queue populated after first death: {}",
        repair_result.actual
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 6. Asymmetric partition + registry — one-way reachability
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn asymmetric_partition_registry_converges_after_heal() {
    // Given: 6 nodes, asymmetric partition: A→B blocked, B→A works.
    // Node 0 (side A) and node 3 (side B) each register a name.
    // After heal, all should converge.
    let config = DistributionSimConfig {
        name: "asymmetric-partition-registry".into(),
        num_nodes: 6,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 0,
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5],
                    asymmetric: true, // A→B blocked, B→A works
                },
            },
            NetworkFault::Heal { round: 40 },
        ],
        action_schedule: vec![
            (12, SimAction::RegisterName { node_idx: 0, name: "from-a".into() }),
            (12, SimAction::RegisterName { node_idx: 3, name: "from-b".into() }),
        ],
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 1,
            suspicion_timeout: 200,
            dead_reprobe_interval: 10,
        },
        ..DistributionSimConfig::default()
    };

    let (trace, nodes) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    // After healing, all nodes should resolve both names
    let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
    for node in &alive_nodes {
        // Side B's name should have been reachable from side A even during partition
        // (B→A works), so "from-b" should propagate to everyone.
        // "from-a" might need post-heal gossip to reach side B.
        assert!(
            node.resolve_name("from-b").is_some(),
            "all nodes should resolve 'from-b' (B→A was always open)"
        );
    }

    // After 60 rounds of healed connectivity, "from-a" should also propagate
    let resolved_a: Vec<_> = alive_nodes
        .iter()
        .filter(|n| n.resolve_name("from-a").is_some())
        .collect();
    assert!(
        resolved_a.len() >= 4,
        "at least 4 of 6 nodes should resolve 'from-a' after partition heals, got {}",
        resolved_a.len()
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 7. Revived node re-registers — new registration overwrites tombstone
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn revived_node_re_registration_overwrites_tombstone() {
    use swactor::actor::ActorAddress;
    let new_actor = ActorAddress::new_random();

    // Given: 5 nodes, node 2 registers "svc", is killed, revived with fresh state,
    // then node 3 re-registers "svc" with a new actor.
    // (Don't kill node 0 since it's the join seed.)
    let config = DistributionSimConfig {
        name: "revive-reregister".into(),
        num_nodes: 5,
        num_rounds: 120,
        ticks_per_round: 3,
        actors_per_node: 0,
        action_schedule: vec![
            (5, SimAction::RegisterName { node_idx: 2, name: "svc".into() }),
            // After death + revive, a different surviving node re-registers
            (60, SimAction::RegisterNameWithActor { node_idx: 3, name: "svc".into(), actor: new_actor }),
        ],
        kill_schedule: vec![(15, 2)],
        revive_schedule: vec![(40, 2)],
        ..DistributionSimConfig::default()
    };

    let (trace, nodes) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    // Then: all alive nodes should resolve "svc" to the new registration
    let alive_nodes: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
    for node in &alive_nodes {
        assert!(
            node.resolve_name("svc").is_some(),
            "all nodes should resolve 'svc' after re-registration by surviving node"
        );
        assert_eq!(
            node.resolve_name("svc").unwrap().0,
            new_actor,
            "all nodes should resolve 'svc' to the new actor"
        );
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 8. Registry convergence is monotonic — divergence doesn't increase
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn registry_convergence_is_monotonic_in_stable_cluster() {
    // Given: 8-node cluster, 4 names registered at round 5, no faults
    let config = DistributionSimConfig {
        name: "registry-monotonic".into(),
        num_nodes: 8,
        num_rounds: 60,
        ticks_per_round: 3,
        actors_per_node: 0,
        action_schedule: vec![
            (5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
            (5, SimAction::RegisterName { node_idx: 2, name: "beta".into() }),
            (5, SimAction::RegisterName { node_idx: 4, name: "gamma".into() }),
            (5, SimAction::RegisterName { node_idx: 6, name: "delta".into() }),
        ],
        ..DistributionSimConfig::default()
    };

    let (trace, _) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    // Measure "divergence" = number of alive nodes with registry_size < 4
    // Once it reaches 0, it should never increase again
    let mut reached_convergence = false;
    let mut post_convergence_divergence = 0;

    for round_snaps in &trace.snapshots_per_round {
        let alive_with_full_registry = round_snaps
            .iter()
            .filter(|(_, s)| s.is_alive && s.registry_size >= 4)
            .count();
        let alive_count = round_snaps.iter().filter(|(_, s)| s.is_alive).count();
        let divergent = alive_count - alive_with_full_registry;

        if divergent == 0 && alive_count > 0 {
            reached_convergence = true;
        } else if reached_convergence && divergent > 0 {
            post_convergence_divergence += 1;
        }
    }

    assert!(
        reached_convergence,
        "registry should converge (all alive nodes see all 4 names)"
    );
    assert_eq!(
        post_convergence_divergence, 0,
        "once converged, registry should not diverge again in a stable cluster"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 9. Large cluster registry stress test
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn large_cluster_registry_converges() {
    // Given: 15-node cluster, 5 names registered on different nodes, 2 deaths
    let config = DistributionSimConfig {
        name: "large-cluster-registry".into(),
        num_nodes: 15,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        action_schedule: vec![
            (5, SimAction::RegisterName { node_idx: 0, name: "alpha".into() }),
            (5, SimAction::RegisterName { node_idx: 3, name: "beta".into() }),
            (5, SimAction::RegisterName { node_idx: 6, name: "gamma".into() }),
            (5, SimAction::RegisterName { node_idx: 9, name: "delta".into() }),
            (5, SimAction::RegisterName { node_idx: 12, name: "epsilon".into() }),
        ],
        kill_schedule: vec![(20, 0), (20, 3)],
        ..DistributionSimConfig::default()
    };

    let (trace, nodes) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    let survivors: Vec<_> = nodes.iter().filter_map(|n| n.as_ref()).collect();
    assert_eq!(survivors.len(), 13, "13 of 15 nodes should survive");

    // Names owned by dead nodes should be tombstoned
    for node in &survivors {
        assert!(
            node.resolve_name("alpha").is_none(),
            "alpha (owned by dead node 0) should be tombstoned"
        );
        assert!(
            node.resolve_name("beta").is_none(),
            "beta (owned by dead node 3) should be tombstoned"
        );
    }

    // Names owned by surviving nodes should resolve
    for node in &survivors {
        for name in &["gamma", "delta", "epsilon"] {
            assert!(
                node.resolve_name(name).is_some(),
                "'{name}' (owned by surviving node) should resolve across 15-node cluster"
            );
        }
    }

    // Registry propagation: all survivors should have all 5 registry entries
    // (2 tombstoned + 3 alive)
    let result = check_registry_propagation(&trace, 5);
    assert!(
        result.passed,
        "all 5 registry entries should propagate in 15-node cluster: {}",
        result.actual
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 10. Three-way partition: cluster splits into 3 groups, heals, converges
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn three_way_partition_heals_and_converges() {
    // Given: 9 nodes split into 3 groups {0,1,2}, {3,4,5}, {6,7,8}.
    // Each group registers a name during the partition. After healing,
    // all 9 nodes should converge on all 3 names.
    let config = DistributionSimConfig {
        name: "three-way-partition".into(),
        num_nodes: 9,
        num_rounds: 140,
        ticks_per_round: 3,
        actors_per_node: 0,
        network_faults: vec![
            // Partition A vs B
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5, 6, 7, 8],
                    asymmetric: false,
                },
            },
            // Partition B vs C (stacks with the above: now 3 groups isolated)
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![3, 4, 5],
                    side_b: vec![6, 7, 8],
                    asymmetric: false,
                },
            },
            NetworkFault::Heal { round: 60 },
        ],
        action_schedule: vec![
            (20, SimAction::RegisterName { node_idx: 0, name: "group-a-svc".into() }),
            (20, SimAction::RegisterName { node_idx: 3, name: "group-b-svc".into() }),
            (20, SimAction::RegisterName { node_idx: 6, name: "group-c-svc".into() }),
        ],
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 1,
            // Must exceed partition duration (50 rounds × 3 ticks = 150 ticks)
            suspicion_timeout: 200,
            dead_reprobe_interval: 10,
        },
        ..DistributionSimConfig::default()
    };

    let (trace, nodes) = run_simulation_with_nodes(config);
    maybe_save_trace(&trace);

    let alive_count = nodes.iter().filter(|n| n.is_some()).count();
    assert_eq!(alive_count, 9, "all 9 nodes should survive the three-way partition");

    // After healing + gossip, all 3 names should be resolvable from every node
    for (i, node) in nodes.iter().enumerate() {
        if let Some(node) = node {
            for name in &["group-a-svc", "group-b-svc", "group-c-svc"] {
                assert!(
                    node.resolve_name(name).is_some(),
                    "node {i} should resolve '{name}' after three-way partition heals"
                );
            }
        }
    }
}