swactor/crates/simulation/tests/cluster_scenarios.rs

#![cfg(feature = "distribution")]
//! Cluster simulation scenarios — breadth-first coverage of failure modes.
//!
//! Inspired by Hashicorp memberlist test suite, FoundationDB simulation testing,
//! and Jepsen/Antithesis fault injection patterns.

use simulation::distribution::properties::{
    analyze, check_accuracy, check_completeness, check_convergence, check_failure_detection,
    check_membership_accuracy,
};
use simulation::distribution::sim::{
    run_simulation, DistributionSimConfig, NetworkFault, Partition,
};
use simulation::distribution::trace::DistributionEventKind;

fn default_config() -> DistributionSimConfig {
    DistributionSimConfig::default()
}

// ────────────────────────────────────────────────────────────────────────────
// 1. Network Partition — symmetric split-brain
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn symmetric_partition_splits_membership_views() {
    // Given: 6 nodes, partition {0,1,2} vs {3,4,5} at round 10
    // Long partitions cause SWIM to declare the other side dead — this is correct behavior.
    // SWIM does not auto-rediscover dead nodes after partition heals.
    let config = DistributionSimConfig {
        name: "symmetric-partition".into(),
        num_nodes: 6,
        num_rounds: 60,
        ticks_per_round: 3,
        actors_per_node: 0,
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5],
                    asymmetric: false,
                },
            },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // After partition, each side should form its own sub-cluster.
    // Each side of 3 nodes should see exactly 2 other members (its own group).
    let last_round = trace.snapshots_per_round.last().unwrap();

    // Side A nodes (0,1,2) should see ≤2 members each (only their partition)
    for idx in 0..3 {
        let snap = &last_round[idx].1;
        assert!(
            snap.is_alive && snap.member_count <= 3,
            "side_a node {} sees {} members, expected ≤3",
            idx,
            snap.member_count
        );
    }

    // Side B nodes (3,4,5) should also see ≤2 members
    for idx in 3..6 {
        let snap = &last_round[idx].1;
        assert!(
            snap.is_alive && snap.member_count <= 3,
            "side_b node {} sees {} members, expected ≤3",
            idx,
            snap.member_count
        );
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 2. Asymmetric partition — one-way communication failure
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn asymmetric_partition_causes_one_sided_suspicion() {
    // Given: 5 nodes, node 4 can send to 0 but 0 can't send to 4
    let config = DistributionSimConfig {
        name: "asymmetric-partition".into(),
        num_nodes: 5,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0],
                    side_b: vec![4],
                    asymmetric: true, // 0→4 blocked, 4→0 works
                },
            },
            NetworkFault::Heal { round: 50 },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // After healing, the cluster should eventually recover
    let last_round = trace.snapshots_per_round.last().unwrap();
    let alive_with_members: Vec<_> = last_round
        .iter()
        .filter(|(_, s)| s.is_alive)
        .map(|(_, s)| s.member_count)
        .collect();

    // All nodes should see at least 3 members after healing
    assert!(
        alive_with_members.iter().all(|&c| c >= 3),
        "all nodes should recover after asymmetric partition heals, got: {alive_with_members:?}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 3. Message loss — random packet dropping
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cluster_converges_under_10_percent_message_loss() {
    // Given: 5 nodes with 10% message loss from the start.
    // 10% loss is significant for SWIM because it can hit both direct probe
    // AND indirect probes in the same cycle, causing false suspicions.
    // Dead reprobe is enabled so false deaths can self-correct — without it,
    // correct death dissemination (via piggyback) causes cascading false deaths
    // that collapse the entire cluster under even modest message loss.
    let config = DistributionSimConfig {
        name: "message-loss-10pct".into(),
        num_nodes: 5,
        num_rounds: 150,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 5,
            indirect_probes: 3,
            suspicion_timeout: 60,
            dead_reprobe_interval: 15,
        },
        network_faults: vec![NetworkFault::SetDropRate {
            round: 1,
            rate: 0.1,
        }],
        ..default_config()
    };

    let trace = run_simulation(config);
    let metrics = analyze(&trace);

    // With 10% loss, SWIM's probe cycle is disrupted enough to cause
    // false suspicions. Dead reprobe allows recovery. We verify:
    // 1. The simulation completes without panic (implicit — we got here)
    // 2. At least partial membership is maintained (some nodes still know about others)
    let result = check_membership_accuracy(&metrics, 0.15);
    assert!(
        result.passed,
        "cluster should maintain some membership under 10% loss: {}",
        result.actual
    );
}

#[test]
fn heavy_message_loss_causes_membership_instability() {
    // Given: 5 nodes with 30% message loss
    // Heavy loss overwhelms SWIM's probe cycle, causing false suspicions.
    // This tests that the protocol degrades but doesn't crash.
    let config = DistributionSimConfig {
        name: "message-loss-30pct".into(),
        num_nodes: 5,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 5,
            indirect_probes: 2,
            suspicion_timeout: 15,
            dead_reprobe_interval: 0,
        },
        network_faults: vec![NetworkFault::SetDropRate {
            round: 1,
            rate: 0.3,
        }],
        ..default_config()
    };

    let trace = run_simulation(config);

    // The simulation should complete without panicking.
    // Under heavy loss, some membership instability is expected.
    let last_round = trace.snapshots_per_round.last().unwrap();
    let alive_count = last_round.iter().filter(|(_, s)| s.is_alive).count();
    assert_eq!(alive_count, 5, "no nodes should actually die");
}

// ────────────────────────────────────────────────────────────────────────────
// 4. Seed node failure — cluster survives without the seed
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cluster_survives_seed_node_death() {
    // Given: 5 nodes, kill the seed (node 0) at round 15
    let config = DistributionSimConfig {
        name: "seed-death".into(),
        num_nodes: 5,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![(15, 0)], // Kill the seed!
        ..default_config()
    };

    let trace = run_simulation(config);

    // Surviving 4 nodes should detect the seed's death
    let result = check_failure_detection(&trace, 4);
    assert!(
        result.passed,
        "survivors should detect seed death: {}",
        result.actual
    );

    // Survivors should still maintain membership among themselves
    let last_round = trace.snapshots_per_round.last().unwrap();
    let survivors: Vec<_> = last_round
        .iter()
        .filter(|(_, s)| s.is_alive)
        .collect();
    assert_eq!(survivors.len(), 4, "4 survivors expected");

    // At least 3 of 4 survivors should see each other
    let well_connected = survivors
        .iter()
        .filter(|(_, s)| s.member_count >= 2)
        .count();
    assert!(
        well_connected >= 3,
        "at least 3 survivors should see ≥2 members, got {well_connected}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 5. Simultaneous multi-node failure
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn simultaneous_two_node_failure_detected() {
    // Given: 7 nodes, kill nodes 2 and 5 simultaneously at round 15
    let config = DistributionSimConfig {
        name: "multi-kill".into(),
        num_nodes: 7,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![(15, 2), (15, 5)],
        ..default_config()
    };

    let trace = run_simulation(config);

    // 5 survivors should see at most 5 members (detecting both deaths)
    let result = check_failure_detection(&trace, 5);
    assert!(
        result.passed,
        "survivors should detect both deaths: {}",
        result.actual
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 6. Cascading sequential failure
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cascading_failures_leave_quorum_intact() {
    // Given: 7 nodes, kill one at r=10, another at r=25, another at r=40
    let config = DistributionSimConfig {
        name: "cascading-failure".into(),
        num_nodes: 7,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![(10, 1), (25, 3), (40, 5)],
        ..default_config()
    };

    let trace = run_simulation(config);

    // 4 survivors should still form a connected cluster
    let last_round = trace.snapshots_per_round.last().unwrap();
    let survivors: Vec<_> = last_round
        .iter()
        .filter(|(_, s)| s.is_alive)
        .collect();
    assert_eq!(survivors.len(), 4, "4 survivors expected");

    // Each survivor should see at most 4 members
    for (name, snap) in &survivors {
        assert!(
            snap.member_count <= 4,
            "{name} sees {} members, expected ≤4",
            snap.member_count
        );
    }
}

// ────────────────────────────────────────────────────────────────────────────
// 7. Large cluster convergence
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cluster_of_fifty_converges() {
    // Given: 50 nodes
    let config = DistributionSimConfig {
        name: "fifty-converges".into(),
        num_nodes: 50,
        num_rounds: 150,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 2,
            suspicion_timeout: 10,
            dead_reprobe_interval: 0,
        },
        ..default_config()
    };

    let trace = run_simulation(config);
    let metrics = analyze(&trace);

    // 50-node cluster should reach ≥80% accuracy
    let result = check_membership_accuracy(&metrics, 0.8);
    assert!(
        result.passed,
        "50-node cluster should converge: {}",
        result.actual
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 8. Rapid churn — nodes joining and dying frequently
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn rapid_churn_maintains_partial_membership() {
    // Given: 8 nodes with rapid kill/revive cycles
    let config = DistributionSimConfig {
        name: "rapid-churn".into(),
        num_nodes: 8,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![
            (10, 2),
            (15, 4),
            (30, 6),
            (45, 3),
        ],
        revive_schedule: vec![
            (25, 2),
            (35, 4),
            (55, 6),
            (65, 3),
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // At end, all nodes should be alive and have some membership view
    let last_round = trace.snapshots_per_round.last().unwrap();
    let alive_count = last_round.iter().filter(|(_, s)| s.is_alive).count();
    assert_eq!(alive_count, 8, "all 8 nodes should be alive at end");

    // At least half should have reasonable membership
    let connected = last_round
        .iter()
        .filter(|(_, s)| s.is_alive && s.member_count >= 3)
        .count();
    assert!(
        connected >= 4,
        "at least 4 of 8 nodes should see ≥3 members after churn, got {connected}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 9. Graceful leave — node announces departure
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn graceful_leave_detected_faster_than_crash() {
    // We can't directly test graceful leave in the current sim harness
    // (leave() is called but doesn't disseminate through ticks in the same way).
    // Instead, test that a crash is detected within a bounded number of rounds.

    // Given: 5 nodes, kill node 2 at round 5
    let config = DistributionSimConfig {
        name: "crash-detection-speed".into(),
        num_nodes: 5,
        num_rounds: 40,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![(5, 2)],
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 2,
            indirect_probes: 1,
            suspicion_timeout: 5,
            dead_reprobe_interval: 0,
        },
        ..default_config()
    };

    let trace = run_simulation(config);

    // Death should be detected by round 20 (suspicion_timeout + margin)
    let mut detected_by_round = None;
    for (round_idx, round_snaps) in trace.snapshots_per_round.iter().enumerate() {
        if round_idx < 5 {
            continue; // Skip rounds before kill
        }
        let all_survivors_see_reduced = round_snaps
            .iter()
            .filter(|(_, s)| s.is_alive)
            .all(|(_, s)| s.member_count <= 4);
        if all_survivors_see_reduced {
            detected_by_round = Some(round_idx + 1);
            break;
        }
    }

    assert!(
        detected_by_round.is_some(),
        "crash should be detected before end of simulation"
    );
    let round = detected_by_round.unwrap();
    assert!(
        round <= 25,
        "crash should be detected by round 25, was detected at round {round}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 10. Partition then kill — compounding failures
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn partition_plus_kill_in_minority_side() {
    // Given: 5 nodes, partition {0,1,2} vs {3,4}, then kill node 3
    let config = DistributionSimConfig {
        name: "partition-plus-kill".into(),
        num_nodes: 5,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 0,
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4],
                    asymmetric: false,
                },
            },
            NetworkFault::Heal { round: 70 },
        ],
        kill_schedule: vec![(20, 3)], // Kill in minority side
        ..default_config()
    };

    let trace = run_simulation(config);

    // After healing, 4 alive nodes should re-converge
    let last_round = trace.snapshots_per_round.last().unwrap();
    let alive_nodes: Vec<_> = last_round
        .iter()
        .filter(|(_, s)| s.is_alive)
        .collect();
    assert_eq!(alive_nodes.len(), 4, "4 nodes should be alive");

    // Majority side {0,1,2} should be well-connected
    let majority_connected = last_round[..3]
        .iter()
        .filter(|(_, s)| s.is_alive && s.member_count >= 2)
        .count();
    assert!(
        majority_connected >= 2,
        "majority partition should maintain connectivity"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 11. Actor resolution under network faults
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn actor_resolution_degrades_during_partition() {
    // Given: 5 nodes with actors, partition at round 10
    let config = DistributionSimConfig {
        name: "actor-resolution-partition".into(),
        num_nodes: 5,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 2,
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1],
                    side_b: vec![2, 3, 4],
                    asymmetric: false,
                },
            },
            NetworkFault::Heal { round: 50 },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // Should still have some successful resolutions (from cache)
    let resolve_events: Vec<_> = trace
        .events
        .iter()
        .filter(|e| matches!(e.kind, DistributionEventKind::ActorResolved { .. }))
        .collect();

    assert!(
        !resolve_events.is_empty(),
        "should still resolve some actors (cached) during partition"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 12. Dissemination completeness — all nodes learn about membership changes
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn membership_changes_disseminate_to_all_nodes() {
    // Given: 10 nodes, kill node 5 at round 20
    let config = DistributionSimConfig {
        name: "dissemination-completeness".into(),
        num_nodes: 10,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![(20, 5)],
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 2,
            suspicion_timeout: 8,
            dead_reprobe_interval: 0,
        },
        ..default_config()
    };

    let trace = run_simulation(config);

    // All 9 survivors should eventually detect the death
    let last_round = trace.snapshots_per_round.last().unwrap();
    let survivors_with_correct_view = last_round
        .iter()
        .filter(|(_, s)| s.is_alive && s.member_count <= 9)
        .count();

    assert!(
        survivors_with_correct_view >= 7,
        "at least 7 of 9 survivors should detect node death, got {survivors_with_correct_view}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 13. Multiple partitions in sequence
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn sequential_partitions_fragment_cluster() {
    // Given: 6 nodes, two sequential partitions.
    // SWIM doesn't auto-rediscover dead-declared nodes, so each partition
    // permanently reduces the membership view of affected nodes.
    let config = DistributionSimConfig {
        name: "sequential-partitions".into(),
        num_nodes: 6,
        num_rounds: 100,
        ticks_per_round: 3,
        actors_per_node: 0,
        network_faults: vec![
            // Partition: {0,1,2} vs {3,4,5}
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5],
                    asymmetric: false,
                },
            },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // Each side should still see its own members
    let last_round = trace.snapshots_per_round.last().unwrap();
    let alive_count = last_round.iter().filter(|(_, s)| s.is_alive).count();
    assert_eq!(alive_count, 6, "all nodes still alive");

    // Side A should maintain internal connectivity
    let side_a_connected = (0..3)
        .filter(|&i| last_round[i].1.member_count >= 1)
        .count();
    assert!(
        side_a_connected >= 2,
        "at least 2 of side_a nodes should see peers, got {side_a_connected}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 14. Message loss then recovery
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn cluster_survives_brief_message_loss() {
    // Given: 5 nodes with 15% loss for a brief window, then clean network.
    // High suspicion timeout + dead reprobe prevents permanent false positives.
    // Without dead reprobe, correct death dissemination causes cascading
    // false deaths that collapse the cluster.
    let config = DistributionSimConfig {
        name: "brief-loss-recovery".into(),
        num_nodes: 5,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 5,
            indirect_probes: 3,
            suspicion_timeout: 60,
            dead_reprobe_interval: 15,
        },
        network_faults: vec![
            NetworkFault::SetDropRate {
                round: 5,
                rate: 0.15,
            },
            NetworkFault::SetDropRate {
                round: 30,
                rate: 0.0,
            },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // After loss stops, most nodes should still be in each other's member lists.
    // Some nodes may have been falsely declared dead during the loss window,
    // but the majority should maintain connectivity.
    let last_round = trace.snapshots_per_round.last().unwrap();
    let well_connected = last_round
        .iter()
        .filter(|(_, s)| s.is_alive && s.member_count >= 2)
        .count();
    assert!(
        well_connected >= 2,
        "at least 2 nodes should see ≥2 members after brief loss, got {well_connected}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// 15. Dead-node reprobe — partition heals, dead nodes recover via reprobe
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn partition_heals_via_dead_reprobe() {
    // Given: 6 nodes, partition {0,1,2} vs {3,4,5} at round 10, heal at round 40.
    // With dead_reprobe_interval enabled, both sides should eventually reprobe
    // the other side's dead-declared nodes, triggering incarnation refutation
    // and recovering the cluster.
    let config = DistributionSimConfig {
        name: "dead-reprobe-recovery".into(),
        num_nodes: 6,
        num_rounds: 120,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 1,
            suspicion_timeout: 5,
            dead_reprobe_interval: 10,
        },
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5],
                    asymmetric: false,
                },
            },
            NetworkFault::Heal { round: 40 },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // After healing + reprobe cycles, nodes should recover cross-partition membership.
    // The reprobe fires every 10 ticks; 80 remaining rounds × 3 ticks = 240 ticks ≫ 10.
    let last_round = trace.snapshots_per_round.last().unwrap();
    let well_connected = last_round
        .iter()
        .filter(|(_, s)| s.is_alive && s.member_count >= 4)
        .count();
    assert!(
        well_connected >= 4,
        "at least 4 of 6 nodes should recover membership after partition heals via reprobe, got {well_connected}"
    );
}

// ────────────────────────────────────────────────────────────────────────────
// SWIM Invariant Tests — formal property checks
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn completeness_all_survivors_detect_failure() {
    // SWIM completeness: every killed node is eventually detected by ALL survivors.
    let config = DistributionSimConfig {
        name: "completeness".into(),
        num_nodes: 7,
        num_rounds: 80,
        ticks_per_round: 3,
        actors_per_node: 0,
        kill_schedule: vec![(15, 3)],
        ..default_config()
    };

    let trace = run_simulation(config);
    // 7 nodes originally alive, kill 1 → survivors should see member_count < 7
    let result = check_completeness(&trace, 15, 7);
    assert!(
        result.passed,
        "completeness failed: {}",
        result.actual
    );
}

#[test]
fn accuracy_no_false_permanent_deaths() {
    // SWIM accuracy: after partition heals with reprobe enabled, no alive node
    // should be permanently declared dead by the majority.
    let config = DistributionSimConfig {
        name: "accuracy".into(),
        num_nodes: 6,
        num_rounds: 120,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 1,
            suspicion_timeout: 5,
            dead_reprobe_interval: 10,
        },
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5],
                    asymmetric: false,
                },
            },
            NetworkFault::Heal { round: 40 },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);
    // All 6 nodes are alive. At least 80% should be well-connected.
    let result = check_accuracy(&trace, 0.8);
    assert!(
        result.passed,
        "accuracy failed: {}",
        result.actual
    );
}

#[test]
fn convergence_after_partition_heal() {
    // SWIM convergence: after partition heals, surviving nodes' member_count
    // values should converge to the same value.
    let config = DistributionSimConfig {
        name: "convergence".into(),
        num_nodes: 6,
        num_rounds: 120,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 1,
            suspicion_timeout: 5,
            dead_reprobe_interval: 10,
        },
        network_faults: vec![
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0, 1, 2],
                    side_b: vec![3, 4, 5],
                    asymmetric: false,
                },
            },
            NetworkFault::Heal { round: 40 },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);
    // After round 60 (20 rounds post-heal), views should converge within ±1
    let result = check_convergence(&trace, 60, 1);
    assert!(
        result.passed,
        "convergence failed: {}",
        result.actual
    );
}

// ────────────────────────────────────────────────────────────────────────────
// Postmortem 3f — suspect→refute race: no false death after refutation
// ────────────────────────────────────────────────────────────────────────────

#[test]
fn suspect_refuted_before_timeout_no_false_death() {
    // Scenario from SWIM flaky test postmortem (recommendation 3f):
    // 3-node cluster, asymmetric partitions make node 1 unreachable by
    // probes from nodes 0 and 2, but node 1's outgoing messages still
    // reach them. Heal before the suspicion timer fires, so node 1 learns
    // about its suspicion via gossip, bumps incarnation, and refutes back
    // to Alive. The suspicion timer then fires but sees Alive (not Suspect)
    // and does nothing — exercising the Bug 1 fix.
    //
    // Timing: partition at round 10 (tick 30), heal at round 20 (tick 60).
    // Suspicion starts ~tick 36. With suspicion_timeout=50, timer fires
    // ~tick 86 — 26 ticks after heal, giving ample time for refutation.
    let config = DistributionSimConfig {
        name: "suspect-refute-race".into(),
        num_nodes: 3,
        num_rounds: 60,
        ticks_per_round: 3,
        actors_per_node: 0,
        swim: distribution::swim::probe::SwimConfig {
            probe_interval: 1,
            probe_timeout: 3,
            indirect_probes: 1,
            suspicion_timeout: 50,
            dead_reprobe_interval: 0, // disabled — refutation must happen before death
        },
        network_faults: vec![
            // Block 0→1 (but 1→0 still works)
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![0],
                    side_b: vec![1],
                    asymmetric: true,
                },
            },
            // Block 2→1 (but 1→2 still works)
            NetworkFault::Partition {
                round: 10,
                partition: Partition {
                    side_a: vec![2],
                    side_b: vec![1],
                    asymmetric: true,
                },
            },
            // Heal all partitions before suspicion timeout fires
            NetworkFault::Heal { round: 20 },
        ],
        ..default_config()
    };

    let trace = run_simulation(config);

    // All 3 nodes must be alive in the final snapshot — no false deaths
    let last_round = trace.snapshots_per_round.last().unwrap();
    for (name, snap) in last_round {
        assert!(
            snap.is_alive,
            "{name} should be alive but was marked dead (false death after refutation)"
        );
    }

    // 100% accuracy: every alive node sees all others as alive
    let result = check_accuracy(&trace, 1.0);
    assert!(
        result.passed,
        "all nodes should see full membership after refutation: {}",
        result.actual
    );
}