From a1f5f581fe746a932b93c2edc9e90fb8227a8ec0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 08:36:52 +0000 Subject: [PATCH 01/10] research: dashboard improvement plan from 10 comparable systems Cycle 0 research complete. Analyzed Erlang Observer, observer_cli, wobserver, Phoenix LiveDashboard, Akka Insights, Ray Dashboard, Orleans Dashboard, tokio-console, RabbitMQ Management, and Consul/Nomad UIs. Identified 8 implementation stages prioritized P0-P2: time-series history, actor detail drill-down, search/filter, worker viz, warning detection, topology, logging, and msg-type breakdown. Full research notes in CLAUDE/notes/ (gitignored, session-local). Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../dashboard-improvements-research.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/development_history/dashboard-improvements-research.md diff --git a/docs/development_history/dashboard-improvements-research.md b/docs/development_history/dashboard-improvements-research.md new file mode 100644 index 0000000..d3db015 --- /dev/null +++ b/docs/development_history/dashboard-improvements-research.md @@ -0,0 +1,28 @@ +# Dashboard Improvements — Research Phase + +## Summary +Researched 10 comparable monitoring/dashboard systems to inform swactor's dashboard improvement plan. + +## Systems Analyzed +- **Actor runtimes**: Erlang Observer (GUI/CLI/Web), Akka Insights, Ray Dashboard, Orleans Dashboard +- **Async/runtime tools**: tokio-console, Lunatic +- **Message/infrastructure**: RabbitMQ Management, Consul UI, Nomad UI +- **Web frameworks**: Phoenix LiveDashboard + +## Key Findings +1. **Time-series history** is table-stakes — every system provides it +2. **Actor detail drill-down** is universal (Observer has 6-tab process info, Orleans has grain state inspection) +3. **Search/filter** exists in every system +4. **Warning/anomaly detection** (tokio-console's lint system) is a high-value differentiator +5. **Topology visualization** (Consul golden metrics, Observer supervision tree) is rare but powerful + +## Implementation Plan +8 feature stages defined (see `CLAUDE/notes/feature-stages/`): +1. Time-Series History Infrastructure +2. Actor Detail Drill-Down +3. Search and Filter +4. Per-Worker Utilization Visualization +5. Warning/Anomaly Detection +6. Actor-to-Actor Message Flow Topology +7. Per-Actor Logging +8. Per-Message-Type Breakdown -- 2.45.2 From 144b4ca5efb65156cc96240769ecb6a847319545 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 08:49:19 +0000 Subject: [PATCH 02/10] feat: time-series history infrastructure with sparklines (Stage 1) Add in-process ring buffer history for per-worker and per-actor stats, with sparkline rendering on both web (SVG) and TUI (ratatui) surfaces. - history.rs: DashboardHistory with bounded VecDeque ring buffers, LRU actor eviction, thread-safe RwLock, 4 unit tests - SSE producer records stats every tick, sends initial history snapshot - /api/history REST endpoint for on-demand history JSON - Web: inline SVG sparklines in worker detail headers (rates + mailbox) - TUI: per-worker Sparkline widgets in overview layout Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../runtime-dashboard/src/dashboard_html.rs | 61 +++- crates/runtime-dashboard/src/history.rs | 343 ++++++++++++++++++ crates/runtime-dashboard/src/lib.rs | 11 + crates/runtime-dashboard/src/server.rs | 25 ++ crates/runtime-dashboard/src/tui/app.rs | 26 ++ crates/runtime-dashboard/src/tui/ui.rs | 42 ++- 6 files changed, 504 insertions(+), 4 deletions(-) create mode 100644 crates/runtime-dashboard/src/history.rs diff --git a/crates/runtime-dashboard/src/dashboard_html.rs b/crates/runtime-dashboard/src/dashboard_html.rs index ec1cc28..06efb72 100644 --- a/crates/runtime-dashboard/src/dashboard_html.rs +++ b/crates/runtime-dashboard/src/dashboard_html.rs @@ -113,6 +113,8 @@ pub const DASHBOARD_HTML: &str = r##" .worker-group-header .wid { font-weight: 700; } .worker-group-header .summary { color: #888; font-size: 11px; } .worker-group-header .toggle { color: #555; font-size: 14px; } + .worker-group-header .sparkline-wrap { display: inline-flex; gap: 8px; margin-left: 12px; } + .worker-group-header .sparkline-wrap svg { vertical-align: middle; } .worker-group-body { display: none; } .worker-group.open .worker-group-body { display: block; } .worker-group-body table { width: 100%; border-collapse: collapse; } @@ -203,6 +205,7 @@ pub const DASHBOARD_HTML: &str = r##" var isReplay = (DASHBOARD_MODE === 'replay'); var lastUptimeMs = null; var lastStatsTime = null; + var workerHistory = {}; // { id: { message_rates: [], mailbox_depths: [] } } var dot = document.getElementById('statusDot'); var uptimeLabel = document.getElementById('uptimeLabel'); @@ -347,6 +350,34 @@ pub const DASHBOARD_HTML: &str = r##" return parts[parts.length - 1]; } + function renderSparklineSvg(data, w, h, color) { + if (!data || data.length < 2) return ''; + var max = Math.max.apply(null, data); + if (max === 0) max = 1; + var step = w / (data.length - 1); + var points = data.map(function(v, i) { + return (i * step).toFixed(1) + ',' + (h - (v / max) * (h - 2) - 1).toFixed(1); + }).join(' '); + return '' + + ''; + } + + function pushHistorySample(stats) { + if (!stats.workers) return; + stats.workers.forEach(function(w) { + if (!workerHistory[w.id]) { + workerHistory[w.id] = { message_rates: [], mailbox_depths: [], prev_msgs: w.messages_processed }; + } + var wh = workerHistory[w.id]; + var rate = w.messages_processed - wh.prev_msgs; + if (rate < 0) rate = 0; + wh.prev_msgs = w.messages_processed; + wh.message_rates.push(rate); + wh.mailbox_depths.push(w.mailbox_depth); + if (wh.message_rates.length > 300) { wh.message_rates.shift(); wh.mailbox_depths.shift(); } + }); + } + function updateWorkerDetails(data) { var container = document.getElementById('workerDetailContainer'); if (!data.workers) return; @@ -377,8 +408,17 @@ pub const DASHBOARD_HTML: &str = r##" var hdr = document.createElement('div'); hdr.className = 'worker-group-header'; var panicHtml = g.info.panics > 0 ? ', ' + g.info.panics + ' panics' : ''; + var wh = workerHistory[wid]; + var sparkHtml = ''; + if (wh) { + sparkHtml = '' + + renderSparklineSvg(wh.message_rates, 80, 16, '#4caf50') + + renderSparklineSvg(wh.mailbox_depths, 80, 16, '#2196f3') + + ''; + } hdr.innerHTML = 'W' + wid + '' + + sparkHtml + '' + g.actors.length + ' actors, ' + g.info.messages_processed.toLocaleString() + ' msgs, mbox ' + g.info.mailbox_depth + panicHtml + '' + '' + (isOpen ? '\u25BC' : '\u25B6') + ''; @@ -467,7 +507,26 @@ pub const DASHBOARD_HTML: &str = r##" var es = new EventSource('/events'); es.addEventListener('stats', function(e) { - try { updateStats(JSON.parse(e.data)); } catch(err) { console.error('stats parse error', err); } + try { + var data = JSON.parse(e.data); + pushHistorySample(data); + updateStats(data); + } catch(err) { console.error('stats parse error', err); } + }); + + es.addEventListener('history', function(e) { + try { + var data = JSON.parse(e.data); + if (data.workers) { + data.workers.forEach(function(w) { + workerHistory[w.id] = { + message_rates: w.message_rates || [], + mailbox_depths: w.mailbox_depths || [], + prev_msgs: 0 + }; + }); + } + } catch(err) { console.error('history parse error', err); } }); es.addEventListener('activity', function(e) { diff --git a/crates/runtime-dashboard/src/history.rs b/crates/runtime-dashboard/src/history.rs new file mode 100644 index 0000000..dd5727d --- /dev/null +++ b/crates/runtime-dashboard/src/history.rs @@ -0,0 +1,343 @@ +//! In-process time-series history for dashboard sparklines and trend detection. +//! +//! Stores bounded ring buffers of per-worker and per-actor stats, sampled at +//! a configurable interval. All data is kept in memory with automatic eviction +//! of the oldest samples when capacity is reached. + +use std::collections::{HashMap, VecDeque}; +use std::sync::RwLock; + +use swactor::actor::ActorAddress; +use swactor::stats::{ActorInfo, RuntimeStats}; + +/// Configuration for history collection. +#[derive(Debug, Clone)] +pub struct HistoryConfig { + /// Maximum samples per worker (default: 300 = 5 min at 1/sec). + pub max_worker_samples: usize, + /// Maximum samples per actor (default: 300). + pub max_actor_samples: usize, + /// Maximum number of actors tracked (LRU eviction). Default: 1000. + pub max_tracked_actors: usize, +} + +impl Default for HistoryConfig { + fn default() -> Self { + Self { + max_worker_samples: 300, + max_actor_samples: 300, + max_tracked_actors: 1000, + } + } +} + +/// Time-series data for a single worker. +#[derive(Debug, Clone)] +pub struct WorkerHistory { + pub message_rates: VecDeque, + pub mailbox_depths: VecDeque, + pub actor_counts: VecDeque, + prev_messages: u64, +} + +impl WorkerHistory { + fn new() -> Self { + Self { + message_rates: VecDeque::new(), + mailbox_depths: VecDeque::new(), + actor_counts: VecDeque::new(), + prev_messages: 0, + } + } + + fn push(&mut self, messages_processed: u64, mailbox_depth: usize, num_actors: usize, cap: usize) { + let rate = messages_processed.saturating_sub(self.prev_messages) as f64; + self.prev_messages = messages_processed; + + push_bounded(&mut self.message_rates, rate, cap); + push_bounded(&mut self.mailbox_depths, mailbox_depth as u64, cap); + push_bounded(&mut self.actor_counts, num_actors as u32, cap); + } +} + +/// Time-series data for a single actor. +#[derive(Debug, Clone)] +pub struct ActorHistory { + pub mailbox_depths: VecDeque, + pub message_rates: VecDeque, + prev_messages: u64, + last_seen_sample: u64, +} + +impl ActorHistory { + fn new(sample_counter: u64) -> Self { + Self { + mailbox_depths: VecDeque::new(), + message_rates: VecDeque::new(), + prev_messages: 0, + last_seen_sample: sample_counter, + } + } + + fn push(&mut self, info: &ActorInfo, cap: usize, sample_counter: u64) { + let rate = info.messages_processed.saturating_sub(self.prev_messages) as f64; + self.prev_messages = info.messages_processed; + self.last_seen_sample = sample_counter; + + push_bounded(&mut self.mailbox_depths, info.mailbox_depth as u64, cap); + push_bounded(&mut self.message_rates, rate, cap); + } +} + +fn push_bounded(buf: &mut VecDeque, val: T, cap: usize) { + if buf.len() >= cap { + buf.pop_front(); + } + buf.push_back(val); +} + +/// Thread-safe history store. Written by the sampler, read by SSE/TUI. +pub struct DashboardHistory { + inner: RwLock, + config: HistoryConfig, +} + +struct HistoryInner { + workers: Vec, + actors: HashMap, + sample_counter: u64, +} + +impl DashboardHistory { + pub fn new(config: HistoryConfig) -> Self { + Self { + inner: RwLock::new(HistoryInner { + workers: Vec::new(), + actors: HashMap::new(), + sample_counter: 0, + }), + config, + } + } + + /// Record a stats snapshot. Called by the sampler thread. + pub fn record(&self, stats: &RuntimeStats) { + let mut inner = self.inner.write().unwrap(); + inner.sample_counter += 1; + let counter = inner.sample_counter; + + // Resize workers vec if needed + while inner.workers.len() < stats.workers.len() { + inner.workers.push(WorkerHistory::new()); + } + + // Record per-worker data + for w in &stats.workers { + if let Some(wh) = inner.workers.get_mut(w.id) { + wh.push( + w.messages_processed, + w.mailbox_depth, + w.num_actors, + self.config.max_worker_samples, + ); + } + } + + // Record per-actor data + for a in &stats.actor_details { + let ah = inner.actors.entry(a.address).or_insert_with(|| ActorHistory::new(counter)); + ah.push(a, self.config.max_actor_samples, counter); + } + + // LRU eviction: remove actors not seen recently if over capacity + if inner.actors.len() > self.config.max_tracked_actors { + let mut entries: Vec<(ActorAddress, u64)> = inner + .actors + .iter() + .map(|(addr, ah)| (*addr, ah.last_seen_sample)) + .collect(); + entries.sort_by_key(|&(_, seen)| seen); + let to_remove = inner.actors.len() - self.config.max_tracked_actors; + for (addr, _) in entries.into_iter().take(to_remove) { + inner.actors.remove(&addr); + } + } + } + + /// Get a snapshot of worker history for rendering sparklines. + /// Returns Vec indexed by worker_id, each containing recent message rates. + pub fn worker_sparklines(&self) -> Vec> { + let inner = self.inner.read().unwrap(); + inner + .workers + .iter() + .map(|wh| wh.message_rates.iter().map(|r| *r as u64).collect()) + .collect() + } + + /// Get worker mailbox depth history. + pub fn worker_mailbox_sparklines(&self) -> Vec> { + let inner = self.inner.read().unwrap(); + inner + .workers + .iter() + .map(|wh| wh.mailbox_depths.iter().copied().collect()) + .collect() + } + + /// Get sparkline data for a specific actor. + pub fn actor_sparkline(&self, addr: &ActorAddress) -> Option<(Vec, Vec)> { + let inner = self.inner.read().unwrap(); + inner.actors.get(addr).map(|ah| { + let mailbox: Vec = ah.mailbox_depths.iter().copied().collect(); + let rates: Vec = ah.message_rates.iter().map(|r| *r as u64).collect(); + (mailbox, rates) + }) + } + + /// Get total sample count (useful for knowing if history is available). + pub fn sample_count(&self) -> u64 { + self.inner.read().unwrap().sample_counter + } + + /// Serialize worker history as JSON for the SSE initial payload. + pub fn worker_history_json(&self) -> String { + let sparklines = self.worker_sparklines(); + let mailbox = self.worker_mailbox_sparklines(); + serde_json::json!({ + "workers": sparklines.iter().enumerate().map(|(i, rates)| { + serde_json::json!({ + "id": i, + "message_rates": rates, + "mailbox_depths": mailbox.get(i).unwrap_or(&Vec::new()), + }) + }).collect::>(), + }) + .to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use swactor::stats::{ActorInfo, WorkerInfo}; + + fn make_stats(workers: Vec<(u64, usize, usize)>, actors: Vec) -> RuntimeStats { + RuntimeStats { + num_workers: workers.len(), + uptime_ms: 0, + actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(), + workers: workers + .into_iter() + .enumerate() + .map(|(id, (msgs, depth, n_actors))| WorkerInfo { + id, + num_actors: n_actors, + mailbox_depth: depth, + messages_processed: msgs, + local_sends: 0, + cross_sends: 0, + inbox_sends: 0, + type_mismatches: 0, + panics: 0, + messages_dropped: 0, + restarts: 0, + stops: 0, + }) + .collect(), + actor_details: actors, + tick_timings: Vec::new(), + } + } + + fn make_actor(id: u8, worker: usize, depth: usize, msgs: u64) -> ActorInfo { + ActorInfo { + address: ActorAddress([id; 32]), + worker_id: worker, + mailbox_depth: depth, + last_msg_type: None, + messages_processed: msgs, + poisoned: false, + } + } + + #[test] + fn worker_rates_accumulate_over_samples() { + let history = DashboardHistory::new(HistoryConfig::default()); + + // First sample: establishes baseline (rate will be the raw value since prev=0) + let stats1 = make_stats(vec![(100, 5, 2)], vec![]); + history.record(&stats1); + + // Second sample: delta = 150 - 100 = 50 + let stats2 = make_stats(vec![(150, 3, 2)], vec![]); + history.record(&stats2); + + let sparklines = history.worker_sparklines(); + assert_eq!(sparklines.len(), 1); + assert_eq!(sparklines[0].len(), 2); + assert_eq!(sparklines[0][0], 100); // first sample: 100 - 0 + assert_eq!(sparklines[0][1], 50); // second sample: 150 - 100 + } + + #[test] + fn bounded_eviction_drops_oldest() { + let config = HistoryConfig { + max_worker_samples: 3, + ..Default::default() + }; + let history = DashboardHistory::new(config); + + for i in 0..5u64 { + let stats = make_stats(vec![(i * 10, 0, 0)], vec![]); + history.record(&stats); + } + + let sparklines = history.worker_sparklines(); + assert_eq!(sparklines[0].len(), 3); // capped at 3 + } + + #[test] + fn actor_lru_eviction_keeps_most_recent() { + let config = HistoryConfig { + max_tracked_actors: 2, + ..Default::default() + }; + let history = DashboardHistory::new(config); + + // Sample 1: actors A and B + let stats1 = make_stats( + vec![(0, 0, 2)], + vec![make_actor(1, 0, 0, 0), make_actor(2, 0, 0, 0)], + ); + history.record(&stats1); + + // Sample 2: actors B and C (A not seen) + let stats2 = make_stats( + vec![(0, 0, 2)], + vec![make_actor(2, 0, 0, 0), make_actor(3, 0, 0, 0)], + ); + history.record(&stats2); + + // A should be evicted (LRU), B and C kept + assert!(history.actor_sparkline(&ActorAddress([1; 32])).is_none()); + assert!(history.actor_sparkline(&ActorAddress([2; 32])).is_some()); + assert!(history.actor_sparkline(&ActorAddress([3; 32])).is_some()); + } + + #[test] + fn actor_rates_track_deltas() { + let history = DashboardHistory::new(HistoryConfig::default()); + + let stats1 = make_stats(vec![(0, 0, 1)], vec![make_actor(1, 0, 5, 100)]); + history.record(&stats1); + + let stats2 = make_stats(vec![(0, 0, 1)], vec![make_actor(1, 0, 3, 175)]); + history.record(&stats2); + + let (mailbox, rates) = history.actor_sparkline(&ActorAddress([1; 32])).unwrap(); + assert_eq!(mailbox, vec![5, 3]); + assert_eq!(rates[0], 100); // first: 100 - 0 + assert_eq!(rates[1], 75); // second: 175 - 100 + } +} diff --git a/crates/runtime-dashboard/src/lib.rs b/crates/runtime-dashboard/src/lib.rs index 79da02a..38b1e41 100644 --- a/crates/runtime-dashboard/src/lib.rs +++ b/crates/runtime-dashboard/src/lib.rs @@ -1,4 +1,5 @@ pub mod collector; +pub mod history; pub mod investigate; pub mod layer; pub mod trace; @@ -27,6 +28,7 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use crate::collector::StatsCollector; +use crate::history::{DashboardHistory, HistoryConfig}; use crate::layer::{now_ms, DashboardLayer, EventStore}; use crate::trace::{RuntimeTrace, TimestampedStats}; @@ -80,6 +82,7 @@ pub struct DashboardHandle { collector: Arc>>>, shutdown: Arc, stats_timeline: Arc>, + history: Arc, recording: bool, #[cfg(feature = "distribution")] distribution: Arc>>>, @@ -115,6 +118,11 @@ impl DashboardHandle { *self.distribution.lock().unwrap() = Some(provider); } + /// Access the time-series history store (for TUI sparklines, etc.). + pub fn history(&self) -> &Arc { + &self.history + } + /// Signal the dashboard to shut down (SSE clients receive "done"). pub fn shutdown(&self) { self.shutdown.store(true, Ordering::Release); @@ -159,6 +167,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { let collector: Arc>>> = Arc::new(Mutex::new(None)); let shutdown = Arc::new(AtomicBool::new(false)); let stats_timeline = Arc::new(ArrayQueue::new(config.record_stats_capacity.max(1))); + let history = Arc::new(DashboardHistory::new(HistoryConfig::default())); #[cfg(feature = "distribution")] let distribution: Arc>>> = @@ -169,6 +178,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { Arc::clone(&runtime), Arc::clone(&collector), Arc::clone(&shutdown), + Arc::clone(&history), config.port, #[cfg(feature = "distribution")] Arc::clone(&distribution), @@ -210,6 +220,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { collector, shutdown, stats_timeline, + history, recording: config.record, #[cfg(feature = "distribution")] distribution, diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index 8d64904..d67ef5a 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -11,6 +11,7 @@ use swactor::runtime::Runtime; use crate::actors_html::ACTORS_HTML; use crate::collector::StatsCollector; use crate::dashboard_html::DASHBOARD_HTML; +use crate::history::DashboardHistory; use crate::layer::EventStore; use crate::trace::RuntimeTrace; @@ -119,6 +120,7 @@ pub(crate) fn spawn_http_server( runtime: Arc>>>, collector: Arc>>>, shutdown: Arc, + history: Arc, port: u16, #[cfg(feature = "distribution")] distribution: Arc>>>, @@ -134,6 +136,7 @@ pub(crate) fn spawn_http_server( let runtime = Arc::clone(&runtime); let collector = Arc::clone(&collector); let shutdown = Arc::clone(&shutdown); + let history = Arc::clone(&history); let cmd_router = Arc::clone(&cmd_router); #[cfg(feature = "distribution")] let distribution = Arc::clone(&distribution); @@ -158,6 +161,7 @@ pub(crate) fn spawn_http_server( Arc::clone(&runtime), Arc::clone(&collector), Arc::clone(&shutdown), + Arc::clone(&history), #[cfg(feature = "distribution")] Arc::clone(&distribution), ); @@ -169,6 +173,9 @@ pub(crate) fn spawn_http_server( Arc::clone(&collector), ); } + "/api/history" => { + handle_history_api(request, Arc::clone(&history)); + } "/api/investigate" => { handle_investigate_api( request, @@ -198,6 +205,7 @@ fn handle_live_sse( runtime: Arc>>>, collector: Arc>>>, shutdown: Arc, + history: Arc, #[cfg(feature = "distribution")] distribution: Arc>>>, ) { @@ -208,6 +216,12 @@ fn handle_live_sse( thread::spawn(move || { let mut cursor: u64 = 0; + // Send initial history snapshot so sparklines render immediately + if history.sample_count() > 0 { + let json = history.worker_history_json(); + let _ = tx.send(format_sse("history", &json)); + } + loop { // Send stats if runtime is available { @@ -217,6 +231,7 @@ fn handle_live_sse( if let Some(col) = collector.lock().unwrap().as_ref() { col.enrich(&mut stats); } + history.record(&stats); let json = serde_json::to_string(&stats).unwrap(); if tx.send(format_sse("stats", &json)).is_err() { return; @@ -353,6 +368,16 @@ fn handle_distribution_api( let _ = request.respond(response); } +fn handle_history_api(request: tiny_http::Request, history: Arc) { + let json = history.worker_history_json(); + let response = tiny_http::Response::from_string(json).with_header( + "Content-Type: application/json" + .parse::() + .unwrap(), + ); + let _ = request.respond(response); +} + fn parse_query_string(url: &str) -> HashMap { let mut params = HashMap::new(); if let Some(qs) = url.split('?').nth(1) { diff --git a/crates/runtime-dashboard/src/tui/app.rs b/crates/runtime-dashboard/src/tui/app.rs index 6901a6c..c90c8fd 100644 --- a/crates/runtime-dashboard/src/tui/app.rs +++ b/crates/runtime-dashboard/src/tui/app.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::time::Instant; use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; @@ -15,6 +16,10 @@ pub struct WorkerView { /// Fraction of bar for each phase group: [processing, delivery, spawns, overhead] pub phase_fractions: [f64; 4], pub panics: u64, + /// Recent message rates for sparkline rendering. + pub sparkline_rates: Vec, + /// Recent mailbox depths for sparkline rendering. + pub sparkline_mailbox: Vec, } /// One row in the actor table. @@ -90,6 +95,10 @@ pub struct App { prev_time: Instant, /// Rolling msg rates (smoothed) msg_rates: Vec, + /// Per-worker sparkline history (message rate deltas). + sparkline_rates: Vec>, + /// Per-worker sparkline history (mailbox depths). + sparkline_mailbox: Vec>, } impl App { @@ -115,6 +124,8 @@ impl App { prev_messages: Vec::new(), prev_time: Instant::now(), msg_rates: Vec::new(), + sparkline_rates: Vec::new(), + sparkline_mailbox: Vec::new(), } } @@ -144,6 +155,8 @@ impl App { if self.prev_messages.len() != stats.workers.len() { self.prev_messages = stats.workers.iter().map(|w| w.messages_processed).collect(); self.msg_rates = vec![0.0; stats.workers.len()]; + self.sparkline_rates.resize_with(stats.workers.len(), VecDeque::new); + self.sparkline_mailbox.resize_with(stats.workers.len(), VecDeque::new); } // Compute per-worker views @@ -163,8 +176,19 @@ impl App { self.msg_rates[i] * 0.6 + rate * 0.4 }; self.msg_rates[i] = smoothed; + + // Track sparkline history + let delta = w.messages_processed.saturating_sub(self.prev_messages[i]); self.prev_messages[i] = w.messages_processed; + let spark_rates = &mut self.sparkline_rates[i]; + if spark_rates.len() >= 60 { spark_rates.pop_front(); } + spark_rates.push_back(delta); + + let spark_mbox = &mut self.sparkline_mailbox[i]; + if spark_mbox.len() >= 60 { spark_mbox.pop_front(); } + spark_mbox.push_back(w.mailbox_depth as u64); + // Load % and phase fractions from tick timings let timings = stats.tick_timings.get(i).map(|v| v.as_slice()).unwrap_or(&[]); let (load_pct, phase_fractions) = compute_load_and_phases(timings); @@ -178,6 +202,8 @@ impl App { load_pct, phase_fractions, panics: w.panics, + sparkline_rates: spark_rates.iter().copied().collect(), + sparkline_mailbox: spark_mbox.iter().copied().collect(), }); } diff --git a/crates/runtime-dashboard/src/tui/ui.rs b/crates/runtime-dashboard/src/tui/ui.rs index 2a51599..c7d9d70 100644 --- a/crates/runtime-dashboard/src/tui/ui.rs +++ b/crates/runtime-dashboard/src/tui/ui.rs @@ -2,7 +2,7 @@ use ratatui::Frame; use ratatui::layout::{Constraint, Layout, Rect}; use ratatui::style::{Color, Modifier, Style}; use ratatui::text::{Line, Span}; -use ratatui::widgets::{Block, Borders, Cell, Paragraph, Row, Table, TableState}; +use ratatui::widgets::{Block, Borders, Cell, Paragraph, Row, Sparkline, Table, TableState}; use super::app::{App, SortColumn, ViewMode}; @@ -20,17 +20,23 @@ pub fn draw(f: &mut Frame, app: &App, table_state: &mut TableState) { fn draw_overview(f: &mut Frame, app: &App, table_state: &mut TableState) { let num_workers = app.workers.len().max(1); + let has_sparkline_data = app.workers.iter().any(|w| w.sparkline_rates.len() > 1); + let sparkline_height = if has_sparkline_data { 4u16 } else { 0u16 }; let chunks = Layout::vertical([ Constraint::Length(num_workers as u16), + Constraint::Length(sparkline_height), Constraint::Length(1), Constraint::Fill(1), ]) .split(f.area()); draw_worker_bars(f, app, chunks[0]); - draw_summary(f, app, chunks[1]); - draw_actor_table(f, app, table_state, chunks[2]); + if has_sparkline_data { + draw_worker_sparklines(f, app, chunks[1]); + } + draw_summary(f, app, chunks[2]); + draw_actor_table(f, app, table_state, chunks[3]); } /// Render htop-style worker bars. @@ -45,6 +51,36 @@ fn draw_worker_bars(f: &mut Frame, app: &App, area: Rect) { f.render_widget(paragraph, area); } +/// Render per-worker sparklines showing message rate trends. +fn draw_worker_sparklines(f: &mut Frame, app: &App, area: Rect) { + if app.workers.is_empty() { + return; + } + // Split area horizontally: one sparkline per worker + let constraints: Vec = app + .workers + .iter() + .map(|_| Constraint::Ratio(1, app.workers.len() as u32)) + .collect(); + let cols = Layout::horizontal(constraints).split(area); + + for (i, w) in app.workers.iter().enumerate() { + if let Some(&col_area) = cols.get(i) { + let block = Block::default() + .borders(Borders::NONE) + .title(Span::styled( + format!(" W{} ", w.id), + Style::default().fg(Color::DarkGray), + )); + let sparkline = Sparkline::default() + .block(block) + .data(&w.sparkline_rates) + .style(Style::default().fg(Color::Green)); + f.render_widget(sparkline, col_area); + } + } +} + fn build_worker_line(w: &super::app::WorkerView, total_width: usize) -> Line<'static> { let id_str = format!("{:>3}", w.id); let suffix = format!( -- 2.45.2 From 5e3a779cba37c9c0a6281c00b20d478a50481fb7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 08:53:00 +0000 Subject: [PATCH 03/10] feat: actor detail drill-down view (Stage 2) Add dedicated actor detail page accessible from both web and TUI surfaces. Web: - /actor/ page with live-updating stats, sparkline charts for message rate and mailbox depth, status badge, worker assignment - Actor addresses in overview and actors pages now link to detail page - Actors page detail panel links to dedicated detail page TUI: - ViewMode::ActorDetail with per-actor sparklines (rate + mailbox) - Enter on actor row opens detail, Esc returns to previous view - Per-actor ring buffer history (60 samples) tracked in App state Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../src/actor_detail_html.rs | 230 ++++++++++++++++++ crates/runtime-dashboard/src/actors_html.rs | 4 +- .../runtime-dashboard/src/dashboard_html.rs | 4 +- crates/runtime-dashboard/src/lib.rs | 1 + crates/runtime-dashboard/src/server.rs | 17 ++ crates/runtime-dashboard/src/tui/app.rs | 55 ++++- crates/runtime-dashboard/src/tui/ui.rs | 94 +++++++ 7 files changed, 396 insertions(+), 9 deletions(-) create mode 100644 crates/runtime-dashboard/src/actor_detail_html.rs diff --git a/crates/runtime-dashboard/src/actor_detail_html.rs b/crates/runtime-dashboard/src/actor_detail_html.rs new file mode 100644 index 0000000..1c7398b --- /dev/null +++ b/crates/runtime-dashboard/src/actor_detail_html.rs @@ -0,0 +1,230 @@ +pub const ACTOR_DETAIL_HTML: &str = r##" + + + + +Actor Detail — Swactor Dashboard + + + +
+
+

+ Swactor Runtime Dashboard + +

+ +
+
+ +
+
+ Overview / Actors / — +
+ +
+
+
Address
—
+
Worker
—
+
Status
—
+
+
+
Last Message Type
—
+
+
+ +
+
0
Messages
+
0
Mailbox
+
0
Msg/s
+
—
Worker Load
+
+ +
+

Message Rate

+ +
+ +
+

Mailbox Depth

+ +
+
+ + + + +"##; diff --git a/crates/runtime-dashboard/src/actors_html.rs b/crates/runtime-dashboard/src/actors_html.rs index 7fb6100..969ba35 100644 --- a/crates/runtime-dashboard/src/actors_html.rs +++ b/crates/runtime-dashboard/src/actors_html.rs @@ -558,8 +558,8 @@ pub const ACTORS_HTML: &str = r##" return; } - document.getElementById('detailAddr').textContent = addrToHex(actor.address); - document.getElementById('detailFullAddr').textContent = focusedAddrHex; + document.getElementById('detailAddr').innerHTML = '' + escapeHtml(addrToHex(actor.address)) + ''; + document.getElementById('detailFullAddr').innerHTML = '' + escapeHtml(focusedAddrHex) + ''; document.getElementById('detailWorker').textContent = 'W' + actor.worker_id; document.getElementById('detailMailbox').textContent = actor.mailbox_depth; document.getElementById('detailMsgCount').textContent = (actor.messages_processed || 0).toLocaleString(); diff --git a/crates/runtime-dashboard/src/dashboard_html.rs b/crates/runtime-dashboard/src/dashboard_html.rs index 06efb72..8004023 100644 --- a/crates/runtime-dashboard/src/dashboard_html.rs +++ b/crates/runtime-dashboard/src/dashboard_html.rs @@ -319,7 +319,7 @@ pub const DASHBOARD_HTML: &str = r##" hex += '\u2026'; } var tr = document.createElement('tr'); - tr.innerHTML = '' + hex + 'W' + wid + ''; + tr.innerHTML = '' + hex + 'W' + wid + ''; tbody.appendChild(tr); }); if (data.actors.length > 200) { @@ -440,7 +440,7 @@ pub const DASHBOARD_HTML: &str = r##" var msgShort = hasMsg ? shortTypeName(a.last_msg_type) : 'none'; var msgClass = hasMsg ? 'msg-type' : 'msg-type none'; var title = hasMsg ? ' title="' + escapeHtml(a.last_msg_type) + '"' : ''; - rows += '' + hex + '' + + rows += '' + hex + '' + '' + a.mailbox_depth + '' + '' + escapeHtml(msgShort) + ''; }); diff --git a/crates/runtime-dashboard/src/lib.rs b/crates/runtime-dashboard/src/lib.rs index 38b1e41..654419b 100644 --- a/crates/runtime-dashboard/src/lib.rs +++ b/crates/runtime-dashboard/src/lib.rs @@ -3,6 +3,7 @@ pub mod history; pub mod investigate; pub mod layer; pub mod trace; +mod actor_detail_html; mod actors_html; mod dashboard_html; mod server; diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index d67ef5a..d3e7bbd 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use swactor::runtime::Runtime; +use crate::actor_detail_html::ACTOR_DETAIL_HTML; use crate::actors_html::ACTORS_HTML; use crate::collector::StatsCollector; use crate::dashboard_html::DASHBOARD_HTML; @@ -192,6 +193,10 @@ pub(crate) fn spawn_http_server( Arc::clone(&distribution), ); } + _ if path.starts_with("/actor/") => { + let hex = &path[7..]; // strip "/actor/" + respond_actor_detail(request, hex); + } _ => respond_404(request), } } @@ -199,6 +204,18 @@ pub(crate) fn spawn_http_server( } } +fn respond_actor_detail(request: tiny_http::Request, hex_addr: &str) { + let html = ACTOR_DETAIL_HTML + .replace("__DASHBOARD_MODE__", "live") + .replace("__ACTOR_ADDR__", hex_addr); + let response = tiny_http::Response::from_string(html).with_header( + "Content-Type: text/html; charset=utf-8" + .parse::() + .unwrap(), + ); + let _ = request.respond(response); +} + fn handle_live_sse( request: tiny_http::Request, store: Arc, diff --git a/crates/runtime-dashboard/src/tui/app.rs b/crates/runtime-dashboard/src/tui/app.rs index c90c8fd..fb5a0d5 100644 --- a/crates/runtime-dashboard/src/tui/app.rs +++ b/crates/runtime-dashboard/src/tui/app.rs @@ -1,4 +1,4 @@ -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::time::Instant; use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; @@ -30,6 +30,10 @@ pub struct ActorRow { pub last_msg_type: Option, pub messages_processed: u64, pub poisoned: bool, + /// Per-actor mailbox sparkline (from local ring buffer). + pub sparkline_mailbox: Vec, + /// Per-actor message rate sparkline. + pub sparkline_rates: Vec, } #[derive(Clone, Copy, PartialEq, Eq)] @@ -67,6 +71,7 @@ impl SortColumn { pub enum ViewMode { Overview, WorkerDetail, + ActorDetail, #[cfg(feature = "distribution")] Distribution, } @@ -84,7 +89,9 @@ pub struct App { pub total_panics: u64, pub num_workers: usize, pub view_mode: ViewMode, + pub prev_view_mode: ViewMode, pub focused_worker: usize, + pub focused_actor: Option, #[cfg(feature = "distribution")] pub distribution: Option, @@ -99,6 +106,8 @@ pub struct App { sparkline_rates: Vec>, /// Per-worker sparkline history (mailbox depths). sparkline_mailbox: Vec>, + /// Per-actor sparkline history: address → (prev_msgs, rates, mailbox_depths). + actor_sparklines: HashMap, VecDeque)>, } impl App { @@ -116,7 +125,9 @@ impl App { total_panics: 0, num_workers: 0, view_mode: ViewMode::Overview, + prev_view_mode: ViewMode::Overview, focused_worker: 0, + focused_actor: None, #[cfg(feature = "distribution")] distribution: None, #[cfg(feature = "distribution")] @@ -126,6 +137,7 @@ impl App { msg_rates: Vec::new(), sparkline_rates: Vec::new(), sparkline_mailbox: Vec::new(), + actor_sparklines: HashMap::new(), } } @@ -136,6 +148,13 @@ impl App { self.distribution = Some(snapshot); } + /// Get the focused actor's data (for actor detail view). + pub fn focused_actor_row(&self) -> Option<&ActorRow> { + self.focused_actor.as_ref().and_then(|addr| { + self.actor_rows.iter().find(|r| r.address == *addr) + }) + } + /// Actor rows filtered to the focused worker (for worker detail view). pub fn focused_actor_rows(&self) -> Vec<&ActorRow> { self.actor_rows @@ -213,9 +232,20 @@ impl App { self.total_mailbox = stats.workers.iter().map(|w| w.mailbox_depth).sum(); self.total_panics = stats.workers.iter().map(|w| w.panics).sum(); - // Build actor table + // Build actor table with sparkline history self.actor_rows.clear(); for a in &stats.actor_details { + let (prev, rates_buf, mbox_buf) = self.actor_sparklines + .entry(a.address) + .or_insert_with(|| (0, VecDeque::new(), VecDeque::new())); + + let rate_delta = a.messages_processed.saturating_sub(*prev); + *prev = a.messages_processed; + if rates_buf.len() >= 60 { rates_buf.pop_front(); } + rates_buf.push_back(rate_delta); + if mbox_buf.len() >= 60 { mbox_buf.pop_front(); } + mbox_buf.push_back(a.mailbox_depth as u64); + self.actor_rows.push(ActorRow { address: a.address, worker_id: a.worker_id, @@ -223,6 +253,8 @@ impl App { last_msg_type: a.last_msg_type.clone(), messages_processed: a.messages_processed, poisoned: a.poisoned, + sparkline_rates: rates_buf.iter().copied().collect(), + sparkline_mailbox: mbox_buf.iter().copied().collect(), }); } self.sort_actors(); @@ -291,6 +323,7 @@ impl App { KeyCode::Tab => { self.view_mode = match self.view_mode { ViewMode::Overview => ViewMode::WorkerDetail, + ViewMode::ActorDetail => ViewMode::Overview, #[cfg(feature = "distribution")] ViewMode::WorkerDetail => ViewMode::Distribution, #[cfg(not(feature = "distribution"))] @@ -306,6 +339,7 @@ impl App { match self.view_mode { ViewMode::Overview => self.handle_key_overview(key), ViewMode::WorkerDetail => self.handle_key_worker_detail(key), + ViewMode::ActorDetail => self.handle_key_actor_detail(key), #[cfg(feature = "distribution")] ViewMode::Distribution => self.handle_key_distribution(key), } @@ -329,11 +363,12 @@ impl App { KeyCode::Home => { self.selected = 0; } KeyCode::End => { self.selected = max; } KeyCode::Enter | KeyCode::Char('l') | KeyCode::Right => { - // Enter worker detail for the selected actor's worker + // Enter actor detail for the selected actor if let Some(row) = self.actor_rows.get(self.selected) { - self.focused_worker = row.worker_id; + self.focused_actor = Some(row.address); + self.prev_view_mode = ViewMode::Overview; + self.view_mode = ViewMode::ActorDetail; } - self.view_mode = ViewMode::WorkerDetail; } _ => {} } @@ -363,6 +398,16 @@ impl App { } } + fn handle_key_actor_detail(&mut self, key: KeyEvent) { + match key.code { + KeyCode::Esc | KeyCode::Char('h') | KeyCode::Left => { + self.view_mode = self.prev_view_mode; + self.focused_actor = None; + } + _ => {} + } + } + #[cfg(feature = "distribution")] fn handle_key_distribution(&mut self, key: KeyEvent) { let max = self diff --git a/crates/runtime-dashboard/src/tui/ui.rs b/crates/runtime-dashboard/src/tui/ui.rs index c7d9d70..21987ae 100644 --- a/crates/runtime-dashboard/src/tui/ui.rs +++ b/crates/runtime-dashboard/src/tui/ui.rs @@ -11,6 +11,7 @@ pub fn draw(f: &mut Frame, app: &App, table_state: &mut TableState) { match app.view_mode { ViewMode::Overview => draw_overview(f, app, table_state), ViewMode::WorkerDetail => draw_worker_detail(f, app, table_state), + ViewMode::ActorDetail => draw_actor_detail(f, app), #[cfg(feature = "distribution")] ViewMode::Distribution => draw_distribution(f, app, table_state), } @@ -479,6 +480,99 @@ fn draw_focused_actor_table(f: &mut Frame, app: &App, table_state: &mut TableSta f.render_stateful_widget(table, area, table_state); } +// ─── Actor Detail View ────────────────────────────────────────────────────── + +fn draw_actor_detail(f: &mut Frame, app: &App) { + let actor = match app.focused_actor_row() { + Some(a) => a, + None => { + let msg = Paragraph::new(" No actor selected. Press Esc to go back.") + .style(Style::default().fg(Color::DarkGray)); + f.render_widget(msg, f.area()); + return; + } + }; + + let chunks = Layout::vertical([ + Constraint::Length(5), // Info card + Constraint::Length(5), // Rate sparkline + Constraint::Length(5), // Mailbox sparkline + Constraint::Length(1), // Help bar + Constraint::Fill(1), // Spacer + ]) + .split(f.area()); + + // Info card + let addr_str = format!("{}", actor.address); + let status = if actor.poisoned { "POISONED" } else { "Healthy" }; + let status_color = if actor.poisoned { Color::Red } else { Color::Green }; + let msg_type = actor.last_msg_type.as_deref() + .map(|s| short_type_name(Some(s))) + .unwrap_or_else(|| "\u{2014}".to_string()); + + let info_lines = vec![ + Line::from(vec![ + Span::styled(" Address: ", Style::default().fg(Color::DarkGray)), + Span::styled(addr_str, Style::default().fg(Color::White).add_modifier(Modifier::BOLD)), + ]), + Line::from(vec![ + Span::styled(" Worker: ", Style::default().fg(Color::DarkGray)), + Span::styled(format!("W{}", actor.worker_id), Style::default().fg(Color::Cyan)), + Span::styled(" Status: ", Style::default().fg(Color::DarkGray)), + Span::styled(status, Style::default().fg(status_color).add_modifier(Modifier::BOLD)), + ]), + Line::from(vec![ + Span::styled(" Messages: ", Style::default().fg(Color::DarkGray)), + Span::styled( + format_num(actor.messages_processed), + Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD), + ), + Span::styled(" Mailbox: ", Style::default().fg(Color::DarkGray)), + Span::styled( + format!("{}", actor.mailbox_depth), + Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD), + ), + Span::styled(" Last Msg: ", Style::default().fg(Color::DarkGray)), + Span::styled(msg_type, Style::default().fg(Color::Green)), + ]), + ]; + + let info_block = Block::default() + .borders(Borders::ALL) + .title(" Actor Detail "); + let info = Paragraph::new(info_lines).block(info_block); + f.render_widget(info, chunks[0]); + + // Rate sparkline + let rate_block = Block::default() + .borders(Borders::ALL) + .title(Span::styled(" Msg Rate ", Style::default().fg(Color::Green))); + let rate_sparkline = Sparkline::default() + .block(rate_block) + .data(&actor.sparkline_rates) + .style(Style::default().fg(Color::Green)); + f.render_widget(rate_sparkline, chunks[1]); + + // Mailbox sparkline + let mbox_block = Block::default() + .borders(Borders::ALL) + .title(Span::styled(" Mailbox Depth ", Style::default().fg(Color::Blue))); + let mbox_sparkline = Sparkline::default() + .block(mbox_block) + .data(&actor.sparkline_mailbox) + .style(Style::default().fg(Color::Blue)); + f.render_widget(mbox_sparkline, chunks[2]); + + // Help bar + let help = Line::from(vec![ + Span::styled( + " Esc/\u{2190}: back Tab: overview q: quit", + Style::default().fg(Color::DarkGray), + ), + ]); + f.render_widget(Paragraph::new(help), chunks[3]); +} + // ─── Distribution View ────────────────────────────────────────────────────── #[cfg(feature = "distribution")] -- 2.45.2 From 41930ef8f4400b083616f42296a0e5f460cea021 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 08:55:07 +0000 Subject: [PATCH 04/10] feat: search and filter for actor tables (Stage 3) TUI: - Press / to enter search mode (vim-style), type to filter actors in real-time by address, message type, or worker ID - Enter locks the filter, Esc clears it - Navigation bounds respect filtered results Web (actors page): - Worker dropdown filter (dynamically populated from live data) - Status filter (All / Healthy / Poisoned) - Mailbox depth threshold filter (min depth) - Text search now also matches message type names Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- crates/runtime-dashboard/src/actors_html.rs | 62 ++++++++++++++++++--- crates/runtime-dashboard/src/tui/app.rs | 61 +++++++++++++++++++- crates/runtime-dashboard/src/tui/ui.rs | 54 +++++++++++++++--- 3 files changed, 157 insertions(+), 20 deletions(-) diff --git a/crates/runtime-dashboard/src/actors_html.rs b/crates/runtime-dashboard/src/actors_html.rs index 969ba35..6f2063a 100644 --- a/crates/runtime-dashboard/src/actors_html.rs +++ b/crates/runtime-dashboard/src/actors_html.rs @@ -187,7 +187,16 @@ pub const ACTORS_HTML: &str = r##"

All Actors

- + + + +
@@ -449,13 +458,29 @@ pub const ACTORS_HTML: &str = r##" function renderActorTable() { var filter = document.getElementById('actorSearch').value.toLowerCase(); - var filtered = currentActors; - if (filter) { - filtered = currentActors.filter(function(a) { + var workerFilter = document.getElementById('workerFilter').value; + var statusFilter = document.getElementById('statusFilter').value; + var minDepthVal = document.getElementById('minDepth').value; + var minDepth = minDepthVal ? parseInt(minDepthVal, 10) : 0; + + var filtered = currentActors.filter(function(a) { + // Text search + if (filter) { var hex = addrToHex(a.address).toLowerCase(); - return hex.indexOf(filter) >= 0 || ('w' + a.worker_id).indexOf(filter) >= 0; - }); - } + var msgType = (a.last_msg_type || '').toLowerCase(); + if (hex.indexOf(filter) < 0 && ('w' + a.worker_id).indexOf(filter) < 0 && msgType.indexOf(filter) < 0) { + return false; + } + } + // Worker filter + if (workerFilter && a.worker_id !== parseInt(workerFilter, 10)) return false; + // Status filter + if (statusFilter === 'healthy' && a.poisoned) return false; + if (statusFilter === 'poisoned' && !a.poisoned) return false; + // Min depth + if (minDepth > 0 && a.mailbox_depth < minDepth) return false; + return true; + }); // Sort filtered.sort(function(a, b) { @@ -647,11 +672,17 @@ pub const ACTORS_HTML: &str = r##" }); } - // ── Search handler ───────────────────────────────────── + // ── Search/filter handlers ───────────────────────────── document.getElementById('actorSearch').addEventListener('keyup', function() { clearTimeout(searchTimer); searchTimer = setTimeout(renderActorTable, 150); }); + document.getElementById('workerFilter').addEventListener('change', renderActorTable); + document.getElementById('statusFilter').addEventListener('change', renderActorTable); + document.getElementById('minDepth').addEventListener('input', function() { + clearTimeout(searchTimer); + searchTimer = setTimeout(renderActorTable, 150); + }); // ── Status helpers ───────────────────────────────────── function setStatus(s) { @@ -697,6 +728,21 @@ pub const ACTORS_HTML: &str = r##" if (!liveAddrs[key]) delete depthHistory[key]; } + // Update worker filter dropdown + var wSelect = document.getElementById('workerFilter'); + var curVal = wSelect.value; + var workerIds = {}; + for (var i = 0; i < currentActors.length; i++) workerIds[currentActors[i].worker_id] = true; + var wids = Object.keys(workerIds).sort(function(a,b) { return +a - +b; }); + wSelect.innerHTML = ''; + wids.forEach(function(wid) { + var opt = document.createElement('option'); + opt.value = wid; + opt.textContent = 'W' + wid; + wSelect.appendChild(opt); + }); + wSelect.value = curVal; + renderActorTable(); if (focusedAddrHex) updateDetailPanel(); } catch(err) { console.error('stats parse error', err); } diff --git a/crates/runtime-dashboard/src/tui/app.rs b/crates/runtime-dashboard/src/tui/app.rs index fb5a0d5..539826e 100644 --- a/crates/runtime-dashboard/src/tui/app.rs +++ b/crates/runtime-dashboard/src/tui/app.rs @@ -92,6 +92,9 @@ pub struct App { pub prev_view_mode: ViewMode, pub focused_worker: usize, pub focused_actor: Option, + pub search_active: bool, + pub search_query: String, + pub search_locked: bool, #[cfg(feature = "distribution")] pub distribution: Option, @@ -128,6 +131,9 @@ impl App { prev_view_mode: ViewMode::Overview, focused_worker: 0, focused_actor: None, + search_active: false, + search_query: String::new(), + search_locked: false, #[cfg(feature = "distribution")] distribution: None, #[cfg(feature = "distribution")] @@ -148,6 +154,21 @@ impl App { self.distribution = Some(snapshot); } + /// Get visible actor rows (filtered by search query if active). + pub fn visible_actor_rows(&self) -> Vec<&ActorRow> { + if self.search_query.is_empty() { + self.actor_rows.iter().collect() + } else { + let q = self.search_query.to_lowercase(); + self.actor_rows.iter().filter(|r| { + let addr = format!("{}", r.address).to_lowercase(); + let msg = r.last_msg_type.as_deref().unwrap_or("").to_lowercase(); + let worker = format!("w{}", r.worker_id); + addr.contains(&q) || msg.contains(&q) || worker.contains(&q) + }).collect() + } + } + /// Get the focused actor's data (for actor detail view). pub fn focused_actor_row(&self) -> Option<&ActorRow> { self.focused_actor.as_ref().and_then(|addr| { @@ -304,12 +325,45 @@ impl App { } pub fn handle_key(&mut self, key: KeyEvent) { + // Search mode input handling + if self.search_active { + match key.code { + KeyCode::Esc => { + self.search_active = false; + if !self.search_locked { + self.search_query.clear(); + } + return; + } + KeyCode::Enter => { + self.search_active = false; + self.search_locked = !self.search_query.is_empty(); + return; + } + KeyCode::Backspace => { + self.search_query.pop(); + return; + } + KeyCode::Char(c) => { + self.search_query.push(c); + return; + } + _ => return, + } + } + // Global keys match key.code { KeyCode::Char('q') => { self.should_quit = true; return; } KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => { self.should_quit = true; return; } + KeyCode::Char('/') => { + self.search_active = true; + self.search_locked = false; + self.search_query.clear(); + return; + } KeyCode::Char('s') => { self.sort_column = self.sort_column.next(); self.sort_actors(); @@ -346,7 +400,8 @@ impl App { } fn handle_key_overview(&mut self, key: KeyEvent) { - let max = if self.actor_rows.is_empty() { 0 } else { self.actor_rows.len() - 1 }; + let visible = self.visible_actor_rows(); + let max = if visible.is_empty() { 0 } else { visible.len() - 1 }; match key.code { KeyCode::Up | KeyCode::Char('k') => { self.selected = self.selected.saturating_sub(1); @@ -363,8 +418,8 @@ impl App { KeyCode::Home => { self.selected = 0; } KeyCode::End => { self.selected = max; } KeyCode::Enter | KeyCode::Char('l') | KeyCode::Right => { - // Enter actor detail for the selected actor - if let Some(row) = self.actor_rows.get(self.selected) { + // Enter actor detail for the selected visible actor + if let Some(&row) = visible.get(self.selected) { self.focused_actor = Some(row.address); self.prev_view_mode = ViewMode::Overview; self.view_mode = ViewMode::ActorDetail; diff --git a/crates/runtime-dashboard/src/tui/ui.rs b/crates/runtime-dashboard/src/tui/ui.rs index 21987ae..13b906d 100644 --- a/crates/runtime-dashboard/src/tui/ui.rs +++ b/crates/runtime-dashboard/src/tui/ui.rs @@ -194,8 +194,33 @@ fn draw_summary(f: &mut Frame, app: &App, area: Rect) { f.render_widget(Paragraph::new(line), area); } -/// Render the actor table with scrolling and selection. +/// Render the actor table with scrolling, selection, and search. fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area: Rect) { + // Split area: optional search bar + table + let has_search = app.search_active || !app.search_query.is_empty(); + let search_height = if has_search { 1u16 } else { 0 }; + let chunks = Layout::vertical([ + Constraint::Length(search_height), + Constraint::Fill(1), + ]) + .split(area); + + // Draw search bar + if has_search { + let search_style = if app.search_active { + Style::default().fg(Color::Yellow) + } else { + Style::default().fg(Color::DarkGray) + }; + let cursor = if app.search_active { "\u{2588}" } else { "" }; + let prefix = if app.search_locked { " [locked] /" } else { " /" }; + let line = Line::from(vec![ + Span::styled(prefix, Style::default().fg(Color::DarkGray)), + Span::styled(format!("{}{}", app.search_query, cursor), search_style), + ]); + f.render_widget(Paragraph::new(line), chunks[0]); + } + let sort_arrow = if app.sort_desc { " \u{25bc}" } else { " \u{25b2}" }; let columns = [ @@ -218,8 +243,8 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area }); let header = Row::new(header_cells).height(1); - let rows: Vec = app - .actor_rows + let visible = app.visible_actor_rows(); + let rows: Vec = visible .iter() .map(|a| actor_row_cells(a)) .collect(); @@ -227,7 +252,22 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area table_state.select(Some(app.selected)); let help_text = - " q: quit \u{2191}\u{2193}: scroll s: sort column r: reverse Tab: worker view Enter: drill in"; + " q: quit /: search \u{2191}\u{2193}: scroll s: sort r: reverse Tab: worker view Enter: detail"; + + let title = if !app.search_query.is_empty() { + format!( + " Actors ({} of {} matching \"{}\") ", + visible.len(), + app.actor_rows.len(), + app.search_query, + ) + } else { + format!( + " Actors (sorted by {}{}) ", + app.sort_column.label(), + sort_arrow + ) + }; let table = Table::new( rows, @@ -243,11 +283,7 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area .block( Block::default() .borders(Borders::ALL) - .title(format!( - " Actors (sorted by {}{}) ", - app.sort_column.label(), - sort_arrow - )) + .title(title) .title_bottom(Line::from(help_text).centered()), ) .row_highlight_style( -- 2.45.2 From 6cfafa04b7bbfb5ae6e8a62f2ef7a0f272035ea1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 08:56:30 +0000 Subject: [PATCH 05/10] feat: per-worker utilization visualization with phase bars (Stage 4) Replace the basic canvas bar chart on the web overview with rich worker cards showing: - Stacked phase-timing bars (processing/delivery/spawns/overhead) - Load percentage derived from tick timing active ratio - Inline sparklines from history (message rate trends) - Actor count, messages processed, mailbox depth stats - Color legend for phase identification Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../runtime-dashboard/src/dashboard_html.rs | 127 ++++++++++++------ 1 file changed, 89 insertions(+), 38 deletions(-) diff --git a/crates/runtime-dashboard/src/dashboard_html.rs b/crates/runtime-dashboard/src/dashboard_html.rs index 8004023..5c544aa 100644 --- a/crates/runtime-dashboard/src/dashboard_html.rs +++ b/crates/runtime-dashboard/src/dashboard_html.rs @@ -77,6 +77,17 @@ pub const DASHBOARD_HTML: &str = r##" .chart-panel { grid-row: span 2; } canvas#workerChart { width: 100%; height: 200px; } + .worker-cards { display: flex; flex-direction: column; gap: 6px; } + .worker-card { + display: flex; align-items: center; gap: 10px; + background: #1c1f2e; border-radius: 4px; padding: 6px 10px; + } + .worker-card .wc-id { font-weight: 700; min-width: 32px; } + .worker-card .wc-bar-wrap { flex: 1; height: 14px; background: #0f1117; border-radius: 2px; overflow: hidden; display: flex; } + .worker-card .wc-bar-seg { height: 100%; } + .worker-card .wc-stats { font-size: 11px; color: #888; min-width: 200px; text-align: right; } + .worker-card .wc-spark { display: inline-flex; gap: 4px; margin-left: 6px; } + .actor-table-wrap { max-height: 200px; overflow-y: auto; } .actor-table-wrap table { width: 100%; border-collapse: collapse; } .actor-table-wrap th, .actor-table-wrap td { @@ -156,9 +167,14 @@ pub const DASHBOARD_HTML: &str = r##"
-

Worker Distribution

- -
+

Worker Utilization

+
+
+ \u25A0 processing + \u25A0 delivery + \u25A0 spawns + \u25A0 overhead +
@@ -243,48 +259,83 @@ pub const DASHBOARD_HTML: &str = r##" } setInterval(updateUptime, 1000); - var canvas = document.getElementById('workerChart'); - var ctx = canvas.getContext('2d'); var colors = ['#4caf50','#2196f3','#ff9800','#f44336','#9c27b0','#00bcd4','#ffeb3b','#e91e63']; - function drawWorkerChart(workers) { - var dpr = window.devicePixelRatio || 1; - var rect = canvas.getBoundingClientRect(); - canvas.width = rect.width * dpr; - canvas.height = rect.height * dpr; - ctx.scale(dpr, dpr); - var W = rect.width, H = rect.height; - ctx.clearRect(0, 0, W, H); + var phaseColors = ['#4caf50', '#2196f3', '#00bcd4', '#f44336']; + // Group tick phases: processing=2, delivery=1+4, spawns=0+3, overhead=5 - if (!workers || workers.length === 0) return; + function computePhases(timings) { + if (!timings || timings.length === 0) return [0.25, 0.25, 0.25, 0.25]; + var sums = [0,0,0,0,0,0]; + var active = 0; + for (var i = 0; i < timings.length; i++) { + var t = timings[i]; + if (t.did_work) active++; + for (var p = 0; p < 6 && p < t.phase_us.length; p++) sums[p] += t.phase_us[p]; + } + var total = sums.reduce(function(a,b) { return a+b; }, 0); + if (total === 0) return [0.25, 0.25, 0.25, 0.25]; + var processing = sums[2] / total; + var delivery = (sums[1] + sums[4]) / total; + var spawns = (sums[0] + sums[3]) / total; + var overhead = sums[5] / total; + var load = timings.length > 0 ? active / timings.length : 0; + return { fracs: [processing, delivery, spawns, overhead], load: load }; + } - var maxActors = Math.max(1, Math.max.apply(null, workers.map(function(w) { return w.num_actors; }))); - var barW = Math.max(8, Math.floor((W - 40) / workers.length) - 6); - var chartH = H - 30; + function renderWorkerCards(data) { + var container = document.getElementById('workerCards'); + if (!data.workers) return; + container.innerHTML = ''; - workers.forEach(function(w, i) { - var x = 20 + i * (barW + 6); - var h = (w.num_actors / maxActors) * (chartH * 0.45); - ctx.fillStyle = colors[i % colors.length]; - ctx.globalAlpha = 0.8; - ctx.fillRect(x, chartH * 0.5 - h, barW, h); + data.workers.forEach(function(w, idx) { + var timings = data.tick_timings ? data.tick_timings[idx] : null; + var phases = computePhases(timings); + var load = phases.load || 0; + var fracs = phases.fracs || [0.25, 0.25, 0.25, 0.25]; - var mh = Math.min(w.mailbox_depth * 2, chartH * 0.4); - ctx.globalAlpha = 0.4; - ctx.fillRect(x, chartH * 0.55, barW, mh); + var card = document.createElement('div'); + card.className = 'worker-card'; - ctx.globalAlpha = 1; - ctx.fillStyle = '#888'; - ctx.font = '10px monospace'; - ctx.textAlign = 'center'; - ctx.fillText('W' + w.id, x + barW / 2, H - 2); + // ID + var idSpan = document.createElement('span'); + idSpan.className = 'wc-id'; + idSpan.style.color = colors[w.id % colors.length]; + idSpan.textContent = 'W' + w.id; + card.appendChild(idSpan); + + // Phase bar + var barWrap = document.createElement('span'); + barWrap.className = 'wc-bar-wrap'; + var filledPct = Math.round(load * 100); + for (var p = 0; p < 4; p++) { + var seg = document.createElement('span'); + seg.className = 'wc-bar-seg'; + seg.style.width = (fracs[p] * filledPct) + '%'; + seg.style.background = phaseColors[p]; + barWrap.appendChild(seg); + } + card.appendChild(barWrap); + + // Sparklines + var sparkWrap = document.createElement('span'); + sparkWrap.className = 'wc-spark'; + var wh = workerHistory[w.id]; + if (wh) { + sparkWrap.innerHTML = renderSparklineSvg(wh.message_rates, 60, 14, '#4caf50'); + } + card.appendChild(sparkWrap); + + // Stats + var statsSpan = document.createElement('span'); + statsSpan.className = 'wc-stats'; + statsSpan.textContent = w.num_actors + ' actors ' + + w.messages_processed.toLocaleString() + ' msgs mbox ' + w.mailbox_depth + + ' ' + Math.round(load * 100) + '%'; + card.appendChild(statsSpan); + + container.appendChild(card); }); - - var legend = document.getElementById('workerLegend'); - legend.innerHTML = workers.map(function(w, i) { - return 'W' + w.id + - ': ' + w.num_actors + ' actors, ' + w.messages_processed + ' msgs, mbox ' + w.mailbox_depth + ''; - }).join('  |  '); } function updateStats(data) { @@ -301,7 +352,7 @@ pub const DASHBOARD_HTML: &str = r##" document.getElementById('statWorkers').textContent = data.num_workers || 0; document.getElementById('statMailbox').textContent = totalMailbox; - drawWorkerChart(data.workers); + renderWorkerCards(data); var tbody = document.getElementById('actorTableBody'); tbody.innerHTML = ''; -- 2.45.2 From 70555c2daf7d127bd4a4c3401f77c9327cd3bbfb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 08:59:38 +0000 Subject: [PATCH 06/10] feat: warning/anomaly detection system (Stage 5) Add automated detection of 6 runtime anomaly types with severity levels: - PoisonedActor (critical): actor panicked - StalledActor (high): no message processing while mailbox > 0 - GrowingMailbox (medium): consecutive increases in mailbox depth - MailboxOverflow (medium): messages dropped (backpressure triggered) - WorkerImbalance (low): one worker has >2x average actor count - EmptyWorker (low): worker has 0 actors while others have many Configurable thresholds with sensible defaults. 7 scenario tests covering all warning types, threshold behavior, and streak reset logic. Web: warning banner at top of overview page with severity-colored alerts TUI: warning count in summary bar (yellow when active) SSE: new "warnings" event sent with each stats tick Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../runtime-dashboard/src/dashboard_html.rs | 19 + crates/runtime-dashboard/src/lib.rs | 1 + crates/runtime-dashboard/src/server.rs | 13 + crates/runtime-dashboard/src/tui/app.rs | 9 + crates/runtime-dashboard/src/tui/ui.rs | 13 +- crates/runtime-dashboard/src/warnings.rs | 391 ++++++++++++++++++ 6 files changed, 444 insertions(+), 2 deletions(-) create mode 100644 crates/runtime-dashboard/src/warnings.rs diff --git a/crates/runtime-dashboard/src/dashboard_html.rs b/crates/runtime-dashboard/src/dashboard_html.rs index 5c544aa..1917297 100644 --- a/crates/runtime-dashboard/src/dashboard_html.rs +++ b/crates/runtime-dashboard/src/dashboard_html.rs @@ -165,6 +165,7 @@ pub const DASHBOARD_HTML: &str = r##"
+

Worker Utilization

@@ -580,6 +581,24 @@ pub const DASHBOARD_HTML: &str = r##" } catch(err) { console.error('history parse error', err); } }); + es.addEventListener('warnings', function(e) { + try { + var warnings = JSON.parse(e.data); + var banner = document.getElementById('warningBanner'); + if (warnings.length === 0) { + banner.style.display = 'none'; + return; + } + banner.style.display = 'block'; + var sevColors = {critical:'#f44336',high:'#ff5722',medium:'#ff9800',low:'#888'}; + var html = warnings.map(function(w) { + var c = sevColors[w.severity] || '#888'; + return '\u26A0 ' + w.description + ''; + }).join('   '); + banner.innerHTML = 'WARNINGS (' + warnings.length + ')   ' + html; + } catch(err) { console.error('warnings parse error', err); } + }); + es.addEventListener('activity', function(e) { try { addLogEvents(JSON.parse(e.data)); } catch(err) { console.error('activity parse error', err); } }); diff --git a/crates/runtime-dashboard/src/lib.rs b/crates/runtime-dashboard/src/lib.rs index 654419b..0f2ed42 100644 --- a/crates/runtime-dashboard/src/lib.rs +++ b/crates/runtime-dashboard/src/lib.rs @@ -3,6 +3,7 @@ pub mod history; pub mod investigate; pub mod layer; pub mod trace; +pub mod warnings; mod actor_detail_html; mod actors_html; mod dashboard_html; diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index d3e7bbd..ee3a580 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -15,6 +15,7 @@ use crate::dashboard_html::DASHBOARD_HTML; use crate::history::DashboardHistory; use crate::layer::EventStore; use crate::trace::RuntimeTrace; +use crate::warnings::{WarningConfig, WarningDetector}; #[cfg(feature = "distribution")] use crate::distribution_collector::DistributionStatsProvider; @@ -232,6 +233,7 @@ fn handle_live_sse( // Spawn producer thread thread::spawn(move || { let mut cursor: u64 = 0; + let mut warning_detector = WarningDetector::new(WarningConfig::default()); // Send initial history snapshot so sparklines render immediately if history.sample_count() > 0 { @@ -249,6 +251,17 @@ fn handle_live_sse( col.enrich(&mut stats); } history.record(&stats); + + // Run warning detection + let warnings = warning_detector.check(&stats); + if !warnings.is_empty() { + if let Ok(wjson) = serde_json::to_string(&warnings) { + if tx.send(format_sse("warnings", &wjson)).is_err() { + return; + } + } + } + let json = serde_json::to_string(&stats).unwrap(); if tx.send(format_sse("stats", &json)).is_err() { return; diff --git a/crates/runtime-dashboard/src/tui/app.rs b/crates/runtime-dashboard/src/tui/app.rs index 539826e..54bdea6 100644 --- a/crates/runtime-dashboard/src/tui/app.rs +++ b/crates/runtime-dashboard/src/tui/app.rs @@ -5,6 +5,8 @@ use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use swactor::actor::ActorAddress; use swactor::stats::{RuntimeStats, TickTiming}; +use crate::warnings::{Warning, WarningConfig, WarningDetector}; + /// Per-worker data prepared for rendering. pub struct WorkerView { pub id: usize, @@ -95,6 +97,7 @@ pub struct App { pub search_active: bool, pub search_query: String, pub search_locked: bool, + pub warnings: Vec, #[cfg(feature = "distribution")] pub distribution: Option, @@ -111,6 +114,7 @@ pub struct App { sparkline_mailbox: Vec>, /// Per-actor sparkline history: address → (prev_msgs, rates, mailbox_depths). actor_sparklines: HashMap, VecDeque)>, + warning_detector: WarningDetector, } impl App { @@ -134,6 +138,7 @@ impl App { search_active: false, search_query: String::new(), search_locked: false, + warnings: Vec::new(), #[cfg(feature = "distribution")] distribution: None, #[cfg(feature = "distribution")] @@ -144,6 +149,7 @@ impl App { sparkline_rates: Vec::new(), sparkline_mailbox: Vec::new(), actor_sparklines: HashMap::new(), + warning_detector: WarningDetector::new(WarningConfig::default()), } } @@ -280,6 +286,9 @@ impl App { } self.sort_actors(); + // Run warning detection + self.warnings = self.warning_detector.check(&stats); + // Clamp selection if !self.actor_rows.is_empty() { self.selected = self.selected.min(self.actor_rows.len() - 1); diff --git a/crates/runtime-dashboard/src/tui/ui.rs b/crates/runtime-dashboard/src/tui/ui.rs index 13b906d..e14d55e 100644 --- a/crates/runtime-dashboard/src/tui/ui.rs +++ b/crates/runtime-dashboard/src/tui/ui.rs @@ -166,7 +166,7 @@ fn draw_summary(f: &mut Frame, app: &App, area: Rect) { Style::default().fg(Color::DarkGray) }; - let line = Line::from(vec![ + let mut spans = vec![ Span::styled(" Workers: ", Style::default().fg(Color::DarkGray)), Span::styled( format!("{}", app.num_workers), @@ -189,8 +189,17 @@ fn draw_summary(f: &mut Frame, app: &App, area: Rect) { ), Span::styled(" Panics: ", Style::default().fg(Color::DarkGray)), Span::styled(format!("{}", app.total_panics), panics_style), - ]); + ]; + if !app.warnings.is_empty() { + spans.push(Span::styled(" Warnings: ", Style::default().fg(Color::DarkGray))); + spans.push(Span::styled( + format!("{}", app.warnings.len()), + Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD), + )); + } + + let line = Line::from(spans); f.render_widget(Paragraph::new(line), area); } diff --git a/crates/runtime-dashboard/src/warnings.rs b/crates/runtime-dashboard/src/warnings.rs new file mode 100644 index 0000000..35400ee --- /dev/null +++ b/crates/runtime-dashboard/src/warnings.rs @@ -0,0 +1,391 @@ +//! Automated anomaly detection for the runtime dashboard. +//! +//! Runs on each stats sample, comparing consecutive snapshots to detect +//! growing mailboxes, stalled actors, worker imbalance, and other conditions. + +use std::collections::HashMap; + +use swactor::actor::ActorAddress; +use swactor::stats::RuntimeStats; + +/// Types of warnings the detector can produce. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum WarningType { + GrowingMailbox, + StalledActor, + PoisonedActor, + WorkerImbalance, + EmptyWorker, + MailboxOverflow, +} + +/// Severity levels for warnings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Severity { + Low, + Medium, + High, + Critical, +} + +/// An active warning. +#[derive(Debug, Clone, serde::Serialize)] +pub struct Warning { + pub warning_type: WarningType, + pub severity: Severity, + pub entity: String, + pub description: String, +} + +/// Configuration for warning thresholds. +#[derive(Debug, Clone)] +pub struct WarningConfig { + /// Consecutive samples with increasing mailbox depth before warning. + pub growing_mailbox_threshold: usize, + /// Consecutive ticks with no message processing while mailbox > 0. + pub stalled_actor_threshold: usize, + /// A worker is "imbalanced" if it has > this ratio times the average load. + pub worker_imbalance_ratio: f64, +} + +impl Default for WarningConfig { + fn default() -> Self { + Self { + growing_mailbox_threshold: 5, + stalled_actor_threshold: 10, + worker_imbalance_ratio: 2.0, + } + } +} + +/// Per-actor tracking state. +struct ActorState { + prev_mailbox: usize, + prev_messages: u64, + growing_streak: usize, + stalled_streak: usize, +} + +/// Warning detection engine. Call `check()` on each stats sample. +pub struct WarningDetector { + config: WarningConfig, + actors: HashMap, +} + +impl WarningDetector { + pub fn new(config: WarningConfig) -> Self { + Self { + config, + actors: HashMap::new(), + } + } + + /// Analyze a stats snapshot and return all active warnings. + pub fn check(&mut self, stats: &RuntimeStats) -> Vec { + let mut warnings = Vec::new(); + + // Track which actors are still alive + let mut live_addrs: std::collections::HashSet = + std::collections::HashSet::new(); + + for actor in &stats.actor_details { + live_addrs.insert(actor.address); + + // Poisoned actor — immediate critical warning + if actor.poisoned { + warnings.push(Warning { + warning_type: WarningType::PoisonedActor, + severity: Severity::Critical, + entity: format!("{}", actor.address), + description: "Actor is poisoned (panicked)".to_string(), + }); + } + + let state = self.actors.entry(actor.address).or_insert(ActorState { + prev_mailbox: actor.mailbox_depth, + prev_messages: actor.messages_processed, + growing_streak: 0, + stalled_streak: 0, + }); + + // Growing mailbox detection + if actor.mailbox_depth > state.prev_mailbox && actor.mailbox_depth > 0 { + state.growing_streak += 1; + } else { + state.growing_streak = 0; + } + + if state.growing_streak >= self.config.growing_mailbox_threshold { + warnings.push(Warning { + warning_type: WarningType::GrowingMailbox, + severity: Severity::Medium, + entity: format!("{}", actor.address), + description: format!( + "Mailbox growing for {} consecutive samples (depth: {})", + state.growing_streak, actor.mailbox_depth, + ), + }); + } + + // Stalled actor detection + if actor.messages_processed == state.prev_messages && actor.mailbox_depth > 0 { + state.stalled_streak += 1; + } else { + state.stalled_streak = 0; + } + + if state.stalled_streak >= self.config.stalled_actor_threshold { + warnings.push(Warning { + warning_type: WarningType::StalledActor, + severity: Severity::High, + entity: format!("{}", actor.address), + description: format!( + "No messages processed for {} ticks with {} pending", + state.stalled_streak, actor.mailbox_depth, + ), + }); + } + + state.prev_mailbox = actor.mailbox_depth; + state.prev_messages = actor.messages_processed; + } + + // Clean up dead actors + self.actors.retain(|addr, _| live_addrs.contains(addr)); + + // Mailbox overflow detection + for w in &stats.workers { + if w.messages_dropped > 0 { + warnings.push(Warning { + warning_type: WarningType::MailboxOverflow, + severity: Severity::Medium, + entity: format!("Worker {}", w.id), + description: format!("{} messages dropped", w.messages_dropped), + }); + } + } + + // Worker imbalance and empty worker detection + if stats.workers.len() > 1 { + let total_actors: usize = stats.workers.iter().map(|w| w.num_actors).sum(); + let avg = total_actors as f64 / stats.workers.len() as f64; + + for w in &stats.workers { + if avg > 0.0 && w.num_actors as f64 > avg * self.config.worker_imbalance_ratio { + warnings.push(Warning { + warning_type: WarningType::WorkerImbalance, + severity: Severity::Low, + entity: format!("Worker {}", w.id), + description: format!( + "{} actors vs {:.0} average ({:.1}x)", + w.num_actors, avg, w.num_actors as f64 / avg, + ), + }); + } + + if w.num_actors == 0 && total_actors > 0 { + warnings.push(Warning { + warning_type: WarningType::EmptyWorker, + severity: Severity::Low, + entity: format!("Worker {}", w.id), + description: "Worker has no actors while others do".to_string(), + }); + } + } + } + + // Sort by severity (critical first) + warnings.sort_by(|a, b| b.severity.cmp(&a.severity)); + warnings + } +} + +#[cfg(test)] +mod tests { + use super::*; + use swactor::stats::{ActorInfo, WorkerInfo}; + + fn make_worker(id: usize, actors: usize, dropped: u64) -> WorkerInfo { + WorkerInfo { + id, + num_actors: actors, + mailbox_depth: 0, + messages_processed: 0, + local_sends: 0, + cross_sends: 0, + inbox_sends: 0, + type_mismatches: 0, + panics: 0, + messages_dropped: dropped, + restarts: 0, + stops: 0, + } + } + + fn make_actor(id: u8, depth: usize, msgs: u64, poisoned: bool) -> ActorInfo { + ActorInfo { + address: ActorAddress([id; 32]), + worker_id: 0, + mailbox_depth: depth, + last_msg_type: None, + messages_processed: msgs, + poisoned, + } + } + + fn make_stats(workers: Vec, actors: Vec) -> RuntimeStats { + RuntimeStats { + num_workers: workers.len(), + uptime_ms: 0, + actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(), + workers, + actor_details: actors, + tick_timings: Vec::new(), + } + } + + #[test] + fn poisoned_actor_triggers_critical_warning() { + let mut detector = WarningDetector::new(WarningConfig::default()); + let stats = make_stats( + vec![make_worker(0, 1, 0)], + vec![make_actor(1, 0, 10, true)], + ); + let warnings = detector.check(&stats); + assert!(warnings.iter().any(|w| w.warning_type == WarningType::PoisonedActor)); + assert!(warnings.iter().any(|w| w.severity == Severity::Critical)); + } + + #[test] + fn growing_mailbox_triggers_after_threshold() { + let config = WarningConfig { + growing_mailbox_threshold: 3, + ..Default::default() + }; + let mut detector = WarningDetector::new(config); + + // 4 samples with increasing mailbox: should trigger at sample 4 + for depth in 1..=4 { + let stats = make_stats( + vec![make_worker(0, 1, 0)], + vec![make_actor(1, depth, 0, false)], + ); + let warnings = detector.check(&stats); + if depth < 4 { + assert!(!warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox), + "should not trigger at depth {}", depth); + } else { + assert!(warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox), + "should trigger at depth {}", depth); + } + } + } + + #[test] + fn growing_mailbox_resets_on_decrease() { + let config = WarningConfig { + growing_mailbox_threshold: 3, + ..Default::default() + }; + let mut detector = WarningDetector::new(config); + + // Grow for 2 samples, then decrease, then grow again + for depth in [1, 2, 1, 2, 3, 4] { + let stats = make_stats( + vec![make_worker(0, 1, 0)], + vec![make_actor(1, depth, 0, false)], + ); + detector.check(&stats); + } + // After 1,2 → streak=2; then 1 → streak=0; then 2,3,4 → streak=3 → triggers + let stats = make_stats( + vec![make_worker(0, 1, 0)], + vec![make_actor(1, 5, 0, false)], + ); + let warnings = detector.check(&stats); + assert!(warnings.iter().any(|w| w.warning_type == WarningType::GrowingMailbox)); + } + + #[test] + fn stalled_actor_triggers_when_not_processing() { + let config = WarningConfig { + stalled_actor_threshold: 3, + ..Default::default() + }; + let mut detector = WarningDetector::new(config); + + // Same messages_processed, nonzero mailbox for 4 ticks + for _ in 0..4 { + let stats = make_stats( + vec![make_worker(0, 1, 0)], + vec![make_actor(1, 5, 100, false)], + ); + let warnings = detector.check(&stats); + // Last one should trigger + if warnings.iter().any(|w| w.warning_type == WarningType::StalledActor) { + return; // test passed + } + } + panic!("expected StalledActor warning"); + } + + #[test] + fn worker_imbalance_detected() { + let mut detector = WarningDetector::new(WarningConfig::default()); + // Worker 0: 10 actors, Worker 1: 1 actor. Avg=5.5, ratio=10/5.5=1.8 + // With ratio threshold 2.0, this should NOT trigger + let stats = make_stats( + vec![make_worker(0, 10, 0), make_worker(1, 1, 0)], + vec![], + ); + let warnings = detector.check(&stats); + assert!(!warnings.iter().any(|w| w.warning_type == WarningType::WorkerImbalance)); + + // Worker 0: 20 actors, Worker 1: 1 actor. Avg=10.5, ratio=20/10.5=1.9 — still no + // Worker 0: 30 actors, Worker 1: 1 actor. Avg=15.5, ratio=30/15.5=1.9 — still no + // Worker 0: 100 actors, Worker 1: 1 actor. Avg=50.5, ratio=100/50.5=1.98 — almost + // Worker 0: 100 actors, Worker 1: 0 actor. Avg=50, ratio=100/50=2.0 — at threshold + + let stats2 = make_stats( + vec![make_worker(0, 100, 0), make_worker(1, 1, 0)], + vec![], + ); + let warnings2 = detector.check(&stats2); + // 100 / 50.5 = 1.98 — not > 2.0 + assert!(!warnings2.iter().any(|w| w.warning_type == WarningType::WorkerImbalance)); + + // Now 200 vs 1: 200/100.5 = ~1.99 — still not. Let's do 300 vs 1: 300/150.5 = ~2.0 + // Actually need > 2x. Let's do 50 vs 1: avg=25.5, ratio=50/25.5=1.96. Nope. + // 10 vs 1 vs 1: avg=4, ratio=10/4=2.5 — triggers! + let stats3 = make_stats( + vec![make_worker(0, 10, 0), make_worker(1, 1, 0), make_worker(2, 1, 0)], + vec![], + ); + let warnings3 = detector.check(&stats3); + assert!(warnings3.iter().any(|w| w.warning_type == WarningType::WorkerImbalance)); + } + + #[test] + fn empty_worker_detected() { + let mut detector = WarningDetector::new(WarningConfig::default()); + let stats = make_stats( + vec![make_worker(0, 5, 0), make_worker(1, 0, 0)], + vec![], + ); + let warnings = detector.check(&stats); + assert!(warnings.iter().any(|w| w.warning_type == WarningType::EmptyWorker)); + } + + #[test] + fn mailbox_overflow_detected() { + let mut detector = WarningDetector::new(WarningConfig::default()); + let stats = make_stats( + vec![make_worker(0, 1, 42)], + vec![], + ); + let warnings = detector.check(&stats); + assert!(warnings.iter().any(|w| w.warning_type == WarningType::MailboxOverflow)); + } +} -- 2.45.2 From 991b2121915899285d50953ba962ab112a85329d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 09:02:07 +0000 Subject: [PATCH 07/10] feat: message flow topology visualization (Stage 6) Add worker-level message flow topology with force-directed graph visualization on a dedicated /topology web page. - topology.rs: worker_topology() derives graph from cross_sends/local_sends stats (TopologyNode, TopologyEdge, TopologySnapshot) - /topology page with interactive force-directed graph layout: - Nodes sized by actor count, colored by worker - Edges show local sends (green self-loops) and cross-worker sends (blue) - Edge thickness proportional to message volume - Physics simulation with repulsion, attraction, and gravity - SSE "topology" event emitted every ~1s (every 5th stats tick) - /api/topology REST endpoint for on-demand snapshot Infrastructure ready for future per-actor topology with core instrumentation. Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- crates/runtime-dashboard/src/lib.rs | 2 + crates/runtime-dashboard/src/server.rs | 47 +++ crates/runtime-dashboard/src/topology.rs | 82 +++++ crates/runtime-dashboard/src/topology_html.rs | 280 ++++++++++++++++++ 4 files changed, 411 insertions(+) create mode 100644 crates/runtime-dashboard/src/topology.rs create mode 100644 crates/runtime-dashboard/src/topology_html.rs diff --git a/crates/runtime-dashboard/src/lib.rs b/crates/runtime-dashboard/src/lib.rs index 0f2ed42..ac7a1c1 100644 --- a/crates/runtime-dashboard/src/lib.rs +++ b/crates/runtime-dashboard/src/lib.rs @@ -8,6 +8,8 @@ mod actor_detail_html; mod actors_html; mod dashboard_html; mod server; +pub mod topology; +mod topology_html; #[cfg(feature = "tui")] pub mod tui; diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index ee3a580..85cd5c7 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -14,6 +14,8 @@ use crate::collector::StatsCollector; use crate::dashboard_html::DASHBOARD_HTML; use crate::history::DashboardHistory; use crate::layer::EventStore; +use crate::topology; +use crate::topology_html::TOPOLOGY_HTML; use crate::trace::RuntimeTrace; use crate::warnings::{WarningConfig, WarningDetector}; @@ -154,6 +156,7 @@ pub(crate) fn spawn_http_server( match path { "/" => respond_html(request, DASHBOARD_HTML, "live"), "/actors" => respond_html(request, ACTORS_HTML, "live"), + "/topology" => respond_html(request, TOPOLOGY_HTML, "live"), #[cfg(feature = "distribution")] "/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"), "/events" => { @@ -178,6 +181,13 @@ pub(crate) fn spawn_http_server( "/api/history" => { handle_history_api(request, Arc::clone(&history)); } + "/api/topology" => { + handle_topology_api( + request, + Arc::clone(&runtime), + Arc::clone(&collector), + ); + } "/api/investigate" => { handle_investigate_api( request, @@ -234,6 +244,7 @@ fn handle_live_sse( thread::spawn(move || { let mut cursor: u64 = 0; let mut warning_detector = WarningDetector::new(WarningConfig::default()); + let mut tick_count: u64 = 0; // Send initial history snapshot so sparklines render immediately if history.sample_count() > 0 { @@ -266,6 +277,17 @@ fn handle_live_sse( if tx.send(format_sse("stats", &json)).is_err() { return; } + + // Send topology every 5th tick (~1/sec) + tick_count += 1; + if tick_count % 5 == 0 { + let topo = topology::worker_topology(&stats); + if let Ok(tjson) = serde_json::to_string(&topo) { + if tx.send(format_sse("topology", &tjson)).is_err() { + return; + } + } + } } } @@ -398,6 +420,31 @@ fn handle_distribution_api( let _ = request.respond(response); } +fn handle_topology_api( + request: tiny_http::Request, + runtime: Arc>>>, + collector: Arc>>>, +) { + let maybe_rt = runtime.lock().unwrap().clone(); + let json = match maybe_rt { + Some(rt) => { + let mut stats = rt.stats(); + if let Some(col) = collector.lock().unwrap().as_ref() { + col.enrich(&mut stats); + } + let topo = topology::worker_topology(&stats); + serde_json::to_string(&topo).unwrap_or_else(|_| "{}".into()) + } + None => "{}".to_string(), + }; + let response = tiny_http::Response::from_string(json).with_header( + "Content-Type: application/json" + .parse::() + .unwrap(), + ); + let _ = request.respond(response); +} + fn handle_history_api(request: tiny_http::Request, history: Arc) { let json = history.worker_history_json(); let response = tiny_http::Response::from_string(json).with_header( diff --git a/crates/runtime-dashboard/src/topology.rs b/crates/runtime-dashboard/src/topology.rs new file mode 100644 index 0000000..c9d78e5 --- /dev/null +++ b/crates/runtime-dashboard/src/topology.rs @@ -0,0 +1,82 @@ +//! Actor-to-actor (and worker-to-worker) message flow topology. +//! +//! Currently derives topology from per-worker cross_sends/local_sends stats. +//! Future: sample-based per-actor source→destination tracking with core instrumentation. + +use swactor::stats::RuntimeStats; + +/// An edge in the topology graph. +#[derive(Debug, Clone, serde::Serialize)] +pub struct TopologyEdge { + pub source: String, + pub target: String, + pub weight: u64, + pub label: String, +} + +/// A node in the topology graph. +#[derive(Debug, Clone, serde::Serialize)] +pub struct TopologyNode { + pub id: String, + pub label: String, + pub actor_count: usize, + pub group: usize, +} + +/// A snapshot of the current topology. +#[derive(Debug, Clone, serde::Serialize)] +pub struct TopologySnapshot { + pub nodes: Vec, + pub edges: Vec, +} + +/// Build a worker-level topology from RuntimeStats. +/// +/// Workers are nodes, edges represent message flow: +/// - Self-loops for local_sends +/// - Cross-edges distributed proportionally (until per-destination tracking exists) +pub fn worker_topology(stats: &RuntimeStats) -> TopologySnapshot { + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + + for w in &stats.workers { + nodes.push(TopologyNode { + id: format!("w{}", w.id), + label: format!("W{}", w.id), + actor_count: w.num_actors, + group: w.id, + }); + + // Local sends = self-loop + if w.local_sends > 0 { + edges.push(TopologyEdge { + source: format!("w{}", w.id), + target: format!("w{}", w.id), + weight: w.local_sends, + label: format!("{} local", w.local_sends), + }); + } + + // Cross sends — without per-destination data, distribute evenly to other workers + if w.cross_sends > 0 && stats.workers.len() > 1 { + let others: Vec<&swactor::stats::WorkerInfo> = + stats.workers.iter().filter(|o| o.id != w.id).collect(); + let per_worker = w.cross_sends / others.len() as u64; + let remainder = w.cross_sends % others.len() as u64; + + for (i, other) in others.iter().enumerate() { + let count = per_worker + if (i as u64) < remainder { 1 } else { 0 }; + if count > 0 { + edges.push(TopologyEdge { + source: format!("w{}", w.id), + target: format!("w{}", other.id), + weight: count, + label: format!("{} cross", count), + }); + } + } + } + } + + TopologySnapshot { nodes, edges } +} diff --git a/crates/runtime-dashboard/src/topology_html.rs b/crates/runtime-dashboard/src/topology_html.rs new file mode 100644 index 0000000..a52ea56 --- /dev/null +++ b/crates/runtime-dashboard/src/topology_html.rs @@ -0,0 +1,280 @@ +pub const TOPOLOGY_HTML: &str = r##" + + + + +Topology — Swactor Dashboard + + + +
+
+

Swactor Runtime Dashboard

+ +
+
+ +
+ +
+ Node size = actor count. Edge thickness = message volume. Green = local sends. Blue = cross-worker sends. +
+
+ + + + +"##; -- 2.45.2 From c7a0bd2523c024c6bed5d68c01024b03ff516052 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Feb 2026 09:11:25 +0000 Subject: [PATCH 08/10] feat(dashboard): per-actor logging with tracing integration (Stage 7) Add actor-level tracing span in worker tick_all to propagate actor_addr to user log events. DashboardLayer now extracts actor_addr from both span extensions and event fields. EventStore gains read_for_actor() for filtered log queries. Web actor detail page gets a live-updating log panel with per-level filter buttons (ERR/WARN/INFO/DBG/TRC) and auto-scroll. TUI actor detail view adds scrollable log panel with j/k navigation and 1-5 keys for level toggling. REST endpoint at /api/logs?actor=&limit=N&level=L. Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .../src/actor_detail_html.rs | 124 ++++++++++++++++++ crates/runtime-dashboard/src/layer.rs | 61 +++++++-- crates/runtime-dashboard/src/server.rs | 29 ++++ crates/runtime-dashboard/src/tui/app.rs | 82 ++++++++++++ crates/runtime-dashboard/src/tui/mod.rs | 7 + crates/runtime-dashboard/src/tui/ui.rs | 79 ++++++++++- src/worker.rs | 3 + 7 files changed, 372 insertions(+), 13 deletions(-) diff --git a/crates/runtime-dashboard/src/actor_detail_html.rs b/crates/runtime-dashboard/src/actor_detail_html.rs index 1c7398b..9f6fdb5 100644 --- a/crates/runtime-dashboard/src/actor_detail_html.rs +++ b/crates/runtime-dashboard/src/actor_detail_html.rs @@ -62,6 +62,40 @@ pub const ACTOR_DETAIL_HTML: &str = r##" .sparkline-panel h3 { font-size: 11px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; } .sparkline-panel svg { width: 100%; height: 50px; } + .logs-panel { + background: #161822; border: 1px solid #2a2d3e; border-radius: 6px; + padding: 14px; margin-bottom: 12px; + } + .logs-panel h3 { + font-size: 11px; color: #888; text-transform: uppercase; letter-spacing: 1px; + margin-bottom: 8px; display: flex; align-items: center; gap: 12px; + } + .level-filter { display: flex; gap: 4px; } + .level-btn { + background: #1e2030; border: 1px solid #2a2d3e; border-radius: 3px; + color: #888; font-size: 10px; padding: 1px 6px; cursor: pointer; + font-family: inherit; + } + .level-btn.active { border-color: #555; color: #fff; } + .level-btn.error { color: #f44336; } + .level-btn.warn { color: #ff9800; } + .level-btn.info { color: #2196f3; } + .level-btn.debug { color: #888; } + .level-btn.trace { color: #555; } + + .log-list { + max-height: 400px; overflow-y: auto; font-size: 11px; line-height: 1.6; + } + .log-entry { display: flex; gap: 8px; padding: 1px 0; border-bottom: 1px solid #1a1c2e; } + .log-time { color: #555; white-space: nowrap; min-width: 80px; } + .log-level { font-weight: 700; min-width: 50px; } + .log-level.ERROR { color: #f44336; } + .log-level.WARN { color: #ff9800; } + .log-level.INFO { color: #2196f3; } + .log-level.DEBUG { color: #888; } + .log-level.TRACE { color: #555; } + .log-msg { color: #e0e0e0; word-break: break-all; } + ::-webkit-scrollbar { width: 6px; } ::-webkit-scrollbar-track { background: #0f1117; } ::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; } @@ -113,6 +147,21 @@ pub const ACTOR_DETAIL_HTML: &str = r##"

Mailbox Depth

+ +
+

+ Logs + (0) +
+ + + + + +
+

+
+