swactor/crates/distribution/tests/lifeguard.rs

//! Behavioral tests for Lifeguard protocol extensions.
//!
//! Tests verify the three Lifeguard mechanisms from the consumer's perspective:
//! 1. Local Health Multiplier (LHM) — degraded nodes get stretched timeouts
//! 2. Dynamic suspect timeout — scales with cluster size
//! 3. Protocol period scaling — probe intervals stretch under load

use std::time::Duration;

use distribution::swim::lifeguard::{HealthMultiplier, LifeguardConfig};

/// Suspicion timeouts are wall-clock now; express the old integer tick-counts
/// as milliseconds so the relative assertions carry over unchanged.
fn ms(n: u64) -> Duration {
    Duration::from_millis(n)
}

// ─── Local Health Multiplier ─────────────────────────────────────────────────

#[test]
fn healthy_node_has_multiplier_of_one() {
    // Given: a freshly created health multiplier
    let hm = HealthMultiplier::new(LifeguardConfig::default());

    // Then: multiplier is 1 (no scaling)
    assert_eq!(hm.multiplier(), 1);
    assert_eq!(hm.score(), 0);
}

#[test]
fn nacks_degrade_health_and_increase_multiplier() {
    // Given: a healthy node
    let mut hm = HealthMultiplier::new(LifeguardConfig::default());

    // When: 3 consecutive nacks occur (no acks)
    hm.record_nack();
    hm.record_nack();
    hm.record_nack();

    // Then: health score is 3, multiplier is 4
    assert_eq!(hm.score(), 3);
    assert_eq!(hm.multiplier(), 4);
}

#[test]
fn acks_improve_health() {
    // Given: a degraded node (score = 3)
    let mut hm = HealthMultiplier::new(LifeguardConfig::default());
    hm.record_nack();
    hm.record_nack();
    hm.record_nack();

    // When: 2 successful acks arrive
    hm.record_ack();
    hm.record_ack();

    // Then: health improves
    assert_eq!(hm.score(), 1);
    assert_eq!(hm.multiplier(), 2);
}

#[test]
fn health_score_cannot_go_below_zero() {
    // Given: a healthy node
    let mut hm = HealthMultiplier::new(LifeguardConfig::default());

    // When: acks arrive despite no prior nacks
    hm.record_ack();
    hm.record_ack();
    hm.record_ack();

    // Then: score stays at 0
    assert_eq!(hm.score(), 0);
    assert_eq!(hm.multiplier(), 1);
}

#[test]
fn health_score_capped_at_max() {
    // Given: a config with max_health_score = 4
    let config = LifeguardConfig {
        max_health_score: 4,
        ..LifeguardConfig::default()
    };
    let mut hm = HealthMultiplier::new(config);

    // When: many nacks occur
    for _ in 0..20 {
        hm.record_nack();
    }

    // Then: score is capped at 4, multiplier at 5
    assert_eq!(hm.score(), 4);
    assert_eq!(hm.multiplier(), 5);
}

// ─── Protocol Period Scaling ─────────────────────────────────────────────────

#[test]
fn healthy_node_uses_base_probe_interval() {
    // Given: a healthy node
    let hm = HealthMultiplier::new(LifeguardConfig::default());

    // When: computing scaled probe interval with base = 10
    let interval = hm.scaled_probe_interval(10);

    // Then: interval is unchanged (multiplier = 1)
    assert_eq!(interval, 10);
}

#[test]
fn degraded_node_stretches_probe_interval() {
    // Given: a node with health score 3 (multiplier = 4)
    let mut hm = HealthMultiplier::new(LifeguardConfig::default());
    hm.record_nack();
    hm.record_nack();
    hm.record_nack();

    // When: computing scaled probe interval with base = 10
    let interval = hm.scaled_probe_interval(10);

    // Then: interval is stretched to 40
    assert_eq!(interval, 40);
}

#[test]
fn degraded_node_stretches_probe_timeout() {
    // Given: a node with health score 2
    let mut hm = HealthMultiplier::new(LifeguardConfig::default());
    hm.record_nack();
    hm.record_nack();

    // When: computing scaled probe timeout with base = 3
    let timeout = hm.scaled_probe_timeout(3);

    // Then: timeout is stretched to 9 (3 * multiplier 3)
    assert_eq!(timeout, 9);
}

// ─── Dynamic Suspect Timeout ─────────────────────────────────────────────────

#[test]
fn suspect_timeout_scales_with_cluster_size() {
    // Given: a healthy node with base_suspicion_timeout = 30ms
    let config = LifeguardConfig {
        base_suspicion_timeout: ms(30),
        min_suspicion_timeout: ms(10),
        max_suspicion_timeout: ms(500),
        ..LifeguardConfig::default()
    };
    let hm = HealthMultiplier::new(config);

    // When: computing dynamic timeout for different cluster sizes
    let timeout_2 = hm.dynamic_suspicion_timeout(2);
    let timeout_8 = hm.dynamic_suspicion_timeout(8);
    let timeout_100 = hm.dynamic_suspicion_timeout(100);

    // Then: larger clusters get longer timeouts (log2 scaling)
    assert!(
        timeout_2 < timeout_8,
        "8-node cluster should have longer timeout than 2-node: {:?} vs {:?}",
        timeout_2, timeout_8
    );
    assert!(
        timeout_8 < timeout_100,
        "100-node cluster should have longer timeout than 8-node: {:?} vs {:?}",
        timeout_8, timeout_100
    );
}

#[test]
fn suspect_timeout_is_clamped_to_min() {
    // Given: a config with min_suspicion_timeout = 50ms and a tiny cluster
    let config = LifeguardConfig {
        base_suspicion_timeout: ms(1),
        min_suspicion_timeout: ms(50),
        max_suspicion_timeout: ms(500),
        ..LifeguardConfig::default()
    };
    let hm = HealthMultiplier::new(config);

    // When: computing for a 2-node cluster (log2(3) ≈ 2, so base*2*1 = 2ms)
    let timeout = hm.dynamic_suspicion_timeout(2);

    // Then: clamped to minimum
    assert_eq!(timeout, ms(50));
}

#[test]
fn suspect_timeout_is_clamped_to_max() {
    // Given: a config with max_suspicion_timeout = 100ms and a huge cluster
    let config = LifeguardConfig {
        base_suspicion_timeout: ms(30),
        min_suspicion_timeout: ms(10),
        max_suspicion_timeout: ms(100),
        ..LifeguardConfig::default()
    };
    let hm = HealthMultiplier::new(config);

    // When: computing for a 10000-node cluster
    let timeout = hm.dynamic_suspicion_timeout(10000);

    // Then: clamped to maximum
    assert_eq!(timeout, ms(100));
}

#[test]
fn degraded_health_further_increases_suspect_timeout() {
    // Given: a config and two nodes — one healthy, one degraded
    let config = LifeguardConfig {
        base_suspicion_timeout: ms(30),
        min_suspicion_timeout: ms(10),
        max_suspicion_timeout: ms(5000),
        ..LifeguardConfig::default()
    };
    let healthy = HealthMultiplier::new(config.clone());
    let mut degraded = HealthMultiplier::new(config);
    degraded.record_nack();
    degraded.record_nack();

    // When: both compute timeout for a 16-node cluster
    let healthy_timeout = healthy.dynamic_suspicion_timeout(16);
    let degraded_timeout = degraded.dynamic_suspicion_timeout(16);

    // Then: degraded node gives itself even more time
    assert!(
        degraded_timeout > healthy_timeout,
        "degraded node ({:?}) should have longer suspect timeout than healthy ({:?})",
        degraded_timeout, healthy_timeout
    );
    // Specifically: healthy = 30 * log2(17) * 1, degraded = 30 * log2(17) * 3
    assert_eq!(degraded_timeout, healthy_timeout * 3);
}

// ─── Stress / Scenario Tests ─────────────────────────────────────────────────

#[test]
fn recovery_from_worst_health_takes_max_acks() {
    // Given: a node at maximum degradation
    let config = LifeguardConfig {
        max_health_score: 8,
        nack_penalty: 1,
        ack_reward: 1,
        ..LifeguardConfig::default()
    };
    let mut hm = HealthMultiplier::new(config);
    for _ in 0..100 {
        hm.record_nack();
    }
    assert_eq!(hm.score(), 8);

    // When: exactly max_health_score acks arrive
    for _ in 0..8 {
        hm.record_ack();
    }

    // Then: fully recovered
    assert_eq!(hm.score(), 0);
    assert_eq!(hm.multiplier(), 1);
}

#[test]
fn mixed_ack_nack_stream_settles_to_moderate_health() {
    // Given: a node receiving alternating acks and nacks (slightly more nacks)
    let config = LifeguardConfig {
        max_health_score: 10,
        nack_penalty: 2,
        ack_reward: 1,
        ..LifeguardConfig::default()
    };
    let mut hm = HealthMultiplier::new(config);

    // When: 100 rounds of alternating nack, ack
    for _ in 0..100 {
        hm.record_nack(); // +2
        hm.record_ack();  // -1
    }

    // Then: score settles near max (nack caps at 10, final ack brings it to 9)
    assert_eq!(hm.score(), 9);
}

#[test]
fn solo_node_gets_minimal_suspect_timeout() {
    // Given: a healthy node in a cluster of size 1
    let config = LifeguardConfig {
        base_suspicion_timeout: ms(30),
        min_suspicion_timeout: ms(15),
        max_suspicion_timeout: ms(500),
        ..LifeguardConfig::default()
    };
    let hm = HealthMultiplier::new(config);

    // When: computing timeout for cluster of 1
    let timeout = hm.dynamic_suspicion_timeout(1);

    // Then: log2(2) = 1, so 30*1*1 = 30 (above min)
    assert_eq!(timeout, ms(30));
}

#[test]
fn empty_cluster_still_returns_valid_timeout() {
    // Given: edge case — cluster size 0
    let config = LifeguardConfig {
        base_suspicion_timeout: ms(30),
        min_suspicion_timeout: ms(15),
        max_suspicion_timeout: ms(500),
        ..LifeguardConfig::default()
    };
    let hm = HealthMultiplier::new(config);

    // When/Then: doesn't panic and returns clamped value
    let timeout = hm.dynamic_suspicion_timeout(0);
    assert!(timeout >= ms(15));
}