swactor/crates/runtime-dashboard/src/warnings.rs

//! Automated anomaly detection for the runtime dashboard.
//!
//! Runs on each stats sample, comparing consecutive snapshots to detect
//! growing mailboxes, stalled actors, worker imbalance, and other conditions.

use std::collections::HashMap;

use swactor::actor::ActorAddress;
use swactor::stats::RuntimeStats;

/// Types of warnings the detector can produce.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub enum WarningType {
    GrowingMailbox,
    StalledActor,
    PoisonedActor,
    WorkerImbalance,
    EmptyWorker,
    MailboxOverflow,
}

/// Severity levels for warnings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Severity {
    Low,
    Medium,
    High,
    Critical,
}

/// An active warning.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Warning {
    pub warning_type: WarningType,
    pub severity: Severity,
    pub entity: String,
    pub description: String,
}

/// Configuration for warning thresholds.
#[derive(Debug, Clone)]
pub struct WarningConfig {
    /// Consecutive samples with increasing mailbox depth before warning.
    pub growing_mailbox_threshold: usize,
    /// Consecutive ticks with no message processing while mailbox > 0.
    pub stalled_actor_threshold: usize,
    /// A worker is "imbalanced" if it has > this ratio times the average load.
    pub worker_imbalance_ratio: f64,
}

impl Default for WarningConfig {
    fn default() -> Self {
        Self {
            growing_mailbox_threshold: 5,
            stalled_actor_threshold: 10,
            worker_imbalance_ratio: 2.0,
        }
    }
}

/// Per-actor tracking state.
struct ActorState {
    prev_mailbox: usize,
    prev_messages: u64,
    growing_streak: usize,
    stalled_streak: usize,
}

/// Warning detection engine. Call `check()` on each stats sample.
pub struct WarningDetector {
    config: WarningConfig,
    actors: HashMap<ActorAddress, ActorState>,
}

impl WarningDetector {
    pub fn new(config: WarningConfig) -> Self {
        Self {
            config,
            actors: HashMap::new(),
        }
    }

    /// Analyze a stats snapshot and return all active warnings.
    pub fn check(&mut self, stats: &RuntimeStats) -> Vec<Warning> {
        let mut warnings = Vec::new();

        // Track which actors are still alive
        let mut live_addrs: std::collections::HashSet<ActorAddress> =
            std::collections::HashSet::new();

        for actor in &stats.actor_details {
            live_addrs.insert(actor.address);

            // Poisoned actor — immediate critical warning
            if actor.poisoned {
                warnings.push(Warning {
                    warning_type: WarningType::PoisonedActor,
                    severity: Severity::Critical,
                    entity: format!("{}", actor.address),
                    description: "Actor is poisoned (panicked)".to_string(),
                });
            }

            let state = self.actors.entry(actor.address).or_insert(ActorState {
                prev_mailbox: actor.mailbox_depth,
                prev_messages: actor.messages_processed,
                growing_streak: 0,
                stalled_streak: 0,
            });

            // Growing mailbox detection
            if actor.mailbox_depth > state.prev_mailbox && actor.mailbox_depth > 0 {
                state.growing_streak += 1;
            } else {
                state.growing_streak = 0;
            }

            if state.growing_streak >= self.config.growing_mailbox_threshold {
                warnings.push(Warning {
                    warning_type: WarningType::GrowingMailbox,
                    severity: Severity::Medium,
                    entity: format!("{}", actor.address),
                    description: format!(
                        "Mailbox growing for {} consecutive samples (depth: {})",
                        state.growing_streak, actor.mailbox_depth,
                    ),
                });
            }

            // Stalled actor detection
            if actor.messages_processed == state.prev_messages && actor.mailbox_depth > 0 {
                state.stalled_streak += 1;
            } else {
                state.stalled_streak = 0;
            }

            if state.stalled_streak >= self.config.stalled_actor_threshold {
                warnings.push(Warning {
                    warning_type: WarningType::StalledActor,
                    severity: Severity::High,
                    entity: format!("{}", actor.address),
                    description: format!(
                        "No messages processed for {} ticks with {} pending",
                        state.stalled_streak, actor.mailbox_depth,
                    ),
                });
            }

            state.prev_mailbox = actor.mailbox_depth;
            state.prev_messages = actor.messages_processed;
        }

        // Clean up dead actors
        self.actors.retain(|addr, _| live_addrs.contains(addr));

        // Mailbox overflow detection
        for w in &stats.workers {
            if w.messages_dropped > 0 {
                warnings.push(Warning {
                    warning_type: WarningType::MailboxOverflow,
                    severity: Severity::Medium,
                    entity: format!("Worker {}", w.id),
                    description: format!("{} messages dropped", w.messages_dropped),
                });
            }
        }

        // Worker imbalance and empty worker detection
        if stats.workers.len() > 1 {
            let total_actors: usize = stats.workers.iter().map(|w| w.num_actors).sum();
            let avg = total_actors as f64 / stats.workers.len() as f64;

            for w in &stats.workers {
                if avg > 0.0 && w.num_actors as f64 > avg * self.config.worker_imbalance_ratio {
                    warnings.push(Warning {
                        warning_type: WarningType::WorkerImbalance,
                        severity: Severity::Low,
                        entity: format!("Worker {}", w.id),
                        description: format!(
                            "{} actors vs {:.0} average ({:.1}x)",
                            w.num_actors, avg, w.num_actors as f64 / avg,
                        ),
                    });
                }

                if w.num_actors == 0 && total_actors > 0 {
                    warnings.push(Warning {
                        warning_type: WarningType::EmptyWorker,
                        severity: Severity::Low,
                        entity: format!("Worker {}", w.id),
                        description: "Worker has no actors while others do".to_string(),
                    });
                }
            }
        }

        // Sort by severity (critical first)
        warnings.sort_by(|a, b| b.severity.cmp(&a.severity));
        warnings
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use swactor::stats::{ActorInfo, WorkerInfo};

    fn make_worker(id: usize, actors: usize, dropped: u64) -> WorkerInfo {
        WorkerInfo {
            id,
            num_actors: actors,
            mailbox_depth: 0,
            messages_processed: 0,
            local_sends: 0,
            cross_sends: 0,
            inbox_sends: 0,
            type_mismatches: 0,
            panics: 0,
            messages_dropped: dropped,
            restarts: 0,
            stops: 0,
        }
    }

    fn make_actor(id: u8, depth: usize, msgs: u64, poisoned: bool) -> ActorInfo {
        ActorInfo {
            address: ActorAddress([id; 32]),
            worker_id: 0,
            mailbox_depth: depth,
            last_msg_type: None,
            messages_processed: msgs,
            poisoned,
            message_type_counts: Vec::new(),
        }
    }

    fn make_stats(workers: Vec<WorkerInfo>, actors: Vec<ActorInfo>) -> RuntimeStats {
        RuntimeStats {
            num_workers: workers.len(),
            uptime_ms: 0,
            actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(),
            workers,
            actor_details: actors,
            tick_timings: Vec::new(),
        }
    }

    #[test]
    fn poisoned_actor_triggers_critical_warning() {
        let mut detector = WarningDetector::new(WarningConfig::default());
        let stats = make_stats(
            vec![make_worker(0, 1, 0)],
            vec![make_actor(1, 0, 10, true)],
        );
        let warnings = detector.check(&stats);
        assert!(warnings.iter().any(|w| w.warning_type == WarningType::PoisonedActor));
        assert!(warnings.iter().any(|w| w.severity == Severity::Critical));
    }

    #[test]
    fn growing_mailbox_triggers_after_threshold() {
        let config = WarningConfig {
            growing_mailbox_threshold: 3,
            ..Default::default()
        };
        let mut detector = WarningDetector::new(config);

        // 4 samples with increasing mailbox: should trigger at sample 4
        for depth in 1..=4 {
            let stats = make_stats(
                vec![make_worker(0, 1, 0)],
                vec![make_actor(1, depth, 0, false)],
            );
            let warnings = detector.check(&stats);
            if depth < 4 {
                assert!(!warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox),
                    "should not trigger at depth {}", depth);
            } else {
                assert!(warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox),
                    "should trigger at depth {}", depth);
            }
        }
    }

    #[test]
    fn growing_mailbox_resets_on_decrease() {
        let config = WarningConfig {
            growing_mailbox_threshold: 3,
            ..Default::default()
        };
        let mut detector = WarningDetector::new(config);

        // Grow for 2 samples, then decrease, then grow again
        for depth in [1, 2, 1, 2, 3, 4] {
            let stats = make_stats(
                vec![make_worker(0, 1, 0)],
                vec![make_actor(1, depth, 0, false)],
            );
            detector.check(&stats);
        }
        // After 1,2 → streak=2; then 1 → streak=0; then 2,3,4 → streak=3 → triggers
        let stats = make_stats(
            vec![make_worker(0, 1, 0)],
            vec![make_actor(1, 5, 0, false)],
        );
        let warnings = detector.check(&stats);
        assert!(warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox));
    }

    #[test]
    fn stalled_actor_triggers_when_not_processing() {
        let config = WarningConfig {
            stalled_actor_threshold: 3,
            ..Default::default()
        };
        let mut detector = WarningDetector::new(config);

        // Same messages_processed, nonzero mailbox for 4 ticks
        for _ in 0..4 {
            let stats = make_stats(
                vec![make_worker(0, 1, 0)],
                vec![make_actor(1, 5, 100, false)],
            );
            let warnings = detector.check(&stats);
            // Last one should trigger
            if warnings.iter().any(|w| w.warning_type == WarningType::StalledActor) {
                return; // test passed
            }
        }
        panic!("expected StalledActor warning");
    }

    #[test]
    fn worker_imbalance_detected() {
        let mut detector = WarningDetector::new(WarningConfig::default());
        // Worker 0: 10 actors, Worker 1: 1 actor. Avg=5.5, ratio=10/5.5=1.8
        // With ratio threshold 2.0, this should NOT trigger
        let stats = make_stats(
            vec![make_worker(0, 10, 0), make_worker(1, 1, 0)],
            vec![],
        );
        let warnings = detector.check(&stats);
        assert!(!warnings.iter().any(|w| w.warning_type == WarningType::WorkerImbalance));

        // Worker 0: 20 actors, Worker 1: 1 actor. Avg=10.5, ratio=20/10.5=1.9 — still no
        // Worker 0: 30 actors, Worker 1: 1 actor. Avg=15.5, ratio=30/15.5=1.9 — still no
        // Worker 0: 100 actors, Worker 1: 1 actor. Avg=50.5, ratio=100/50.5=1.98 — almost
        // Worker 0: 100 actors, Worker 1: 0 actor. Avg=50, ratio=100/50=2.0 — at threshold

        let stats2 = make_stats(
            vec![make_worker(0, 100, 0), make_worker(1, 1, 0)],
            vec![],
        );
        let warnings2 = detector.check(&stats2);
        // 100 / 50.5 = 1.98 — not > 2.0
        assert!(!warnings2.iter().any(|w| w.warning_type == WarningType::WorkerImbalance));

        // Now 200 vs 1: 200/100.5 = ~1.99 — still not. Let's do 300 vs 1: 300/150.5 = ~2.0
        // Actually need > 2x. Let's do 50 vs 1: avg=25.5, ratio=50/25.5=1.96. Nope.
        // 10 vs 1 vs 1: avg=4, ratio=10/4=2.5 — triggers!
        let stats3 = make_stats(
            vec![make_worker(0, 10, 0), make_worker(1, 1, 0), make_worker(2, 1, 0)],
            vec![],
        );
        let warnings3 = detector.check(&stats3);
        assert!(warnings3.iter().any(|w| w.warning_type == WarningType::WorkerImbalance));
    }

    #[test]
    fn empty_worker_detected() {
        let mut detector = WarningDetector::new(WarningConfig::default());
        let stats = make_stats(
            vec![make_worker(0, 5, 0), make_worker(1, 0, 0)],
            vec![],
        );
        let warnings = detector.check(&stats);
        assert!(warnings.iter().any(|w| w.warning_type == WarningType::EmptyWorker));
    }

    #[test]
    fn mailbox_overflow_detected() {
        let mut detector = WarningDetector::new(WarningConfig::default());
        let stats = make_stats(
            vec![make_worker(0, 1, 42)],
            vec![],
        );
        let warnings = detector.check(&stats);
        assert!(warnings.iter().any(|w| w.warning_type == WarningType::MailboxOverflow));
    }
}