feat: wasm runner actor skeleton (#32)

Lay down the wasm-actor host, a frontend-agnostic command layer, and a
distribution registry.

- command (new crate): CommandRouter dispatching to built-in inspection handlers
  (overview/workers/actors) plus user-registered handlers, with line and
  query-param parsers; built for REPL/REST/TUI/WebSocket frontends.
- wasm-actor (new crate): skeleton host — WasmActor, Builder, Engine, error
  types — with echo/double/silent guest fixtures and integration tests.
- distribution: add Registry (member catalog + lookups) and Snapshot, with tests.
- core: extend the worker watch API; add watch_api integration tests.

Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
zacheryasc 2026-02-13 07:42:44 +00:00
parent 3c29293945
commit 4e56590f05
43 changed files with 5049 additions and 644 deletions

4
.gitignore vendored
View file

@ -11,3 +11,7 @@ corpus
**/deps.html **/deps.html
docs/architecture.dot docs/architecture.dot
docs/architecture.html docs/architecture.html
# Claude session files
CLAUDE/
.claude/

1083
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,5 +1,5 @@
[workspace] [workspace]
members = [".", "crates/python", "crates/wasm", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std"] members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std", "crates/command"]
exclude = ["tools/depgraph"] exclude = ["tools/depgraph"]
[package] [package]

View file

@ -0,0 +1,9 @@
[package]
name = "swactor-command"
version = "0.1.0"
edition = "2024"
[dependencies]
swactor = { path = "../..", features = ["serde"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"

View file

@ -0,0 +1,561 @@
//! Built-in command handlers for runtime inspection and management.
//!
//! Extracted from `crates/runtime-dashboard/src/investigate.rs`.
use std::collections::HashMap;
use std::time::{Duration, Instant};
use swactor::actor::ActorAddress;
use swactor::stats::TickTiming;
use crate::{CommandContext, CommandHandler, CommandMeta, CommandResponse};
// ─── Arg helpers ─────────────────────────────────────────────────────────────
fn arg_str<'a>(args: &'a HashMap<String, serde_json::Value>, key: &str) -> Option<&'a str> {
args.get(key).and_then(|v| v.as_str())
}
fn arg_usize(args: &HashMap<String, serde_json::Value>, key: &str) -> Option<usize> {
args.get(key).and_then(|v| {
v.as_u64()
.map(|n| n as usize)
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
})
}
fn arg_f64(args: &HashMap<String, serde_json::Value>, key: &str) -> Option<f64> {
args.get(key).and_then(|v| {
v.as_f64()
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
})
}
// ─── Display helpers ─────────────────────────────────────────────────────────
fn format_addr(addr: &ActorAddress) -> String {
format!("{addr}")
}
fn full_hex(addr: &ActorAddress) -> String {
addr.0.iter().map(|b| format!("{b:02x}")).collect()
}
// ─── Phase breakdown helper ──────────────────────────────────────────────────
fn compute_phase_breakdown(timings: &[TickTiming]) -> serde_json::Value {
if timings.is_empty() {
return serde_json::json!({
"ticks": 0,
"active_pct": 0.0,
"avg_tick_us": 0.0,
"phases_us": [0, 0, 0, 0, 0, 0],
"phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
});
}
let n = timings.len();
let active = timings.iter().filter(|t| t.did_work).count();
let active_pct = (active as f64 / n as f64) * 100.0;
let mut phase_sums = [0u64; 6];
for t in timings {
for (i, &us) in t.phase_us.iter().enumerate() {
phase_sums[i] += us;
}
}
let total_us: u64 = phase_sums.iter().sum();
let avg_tick_us = total_us as f64 / n as f64;
let phases_pct: Vec<f64> = if total_us == 0 {
vec![0.0; 6]
} else {
phase_sums
.iter()
.map(|&s| (s as f64 / total_us as f64) * 100.0)
.collect()
};
serde_json::json!({
"ticks": n,
"active_pct": active_pct,
"avg_tick_us": avg_tick_us,
"phases_us": phase_sums,
"phases_pct": phases_pct,
})
}
// ─── Read Commands ───────────────────────────────────────────────────────────
pub struct OverviewCommand;
impl CommandHandler for OverviewCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "overview",
description: "Summary: worker count, actor count, total messages, mailbox depth, panics",
usage: "overview",
is_write: false,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.enriched_stats();
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum();
let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum();
let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum();
let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum();
let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum();
let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum();
CommandResponse::ok(
"overview",
serde_json::json!({
"workers": stats.num_workers,
"actors": stats.actor_details.len(),
"total_messages_processed": total_msgs,
"total_mailbox_depth": total_mailbox,
"total_panics": total_panics,
"total_type_mismatches": total_type_mismatches,
"sends": {
"local": total_local,
"cross_worker": total_cross,
"inbox": total_inbox,
},
}),
)
}
}
pub struct WorkersCommand;
impl CommandHandler for WorkersCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "workers",
description: "Per-worker stats: actors, mailbox depth, messages, sends, panics",
usage: "workers",
is_write: false,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.stats();
let workers: Vec<_> = stats
.workers
.iter()
.map(|w| {
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
})
})
.collect();
CommandResponse::ok("workers", workers)
}
}
pub struct WorkerCommand;
impl CommandHandler for WorkerCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "worker",
description: "Single worker detail with tick-phase timing breakdown",
usage: "worker <id>",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let id = match arg_usize(args, "id") {
Some(id) => id,
None => return CommandResponse::err("worker", "usage: worker <id>"),
};
let stats = ctx.enriched_stats();
let w = match stats.workers.iter().find(|w| w.id == id) {
Some(w) => w,
None => {
return CommandResponse::err(
"worker",
format!("worker {id} not found (have 0..{})", stats.num_workers),
)
}
};
let timings = stats.tick_timings.get(id).cloned().unwrap_or_default();
let phase_breakdown = compute_phase_breakdown(&timings);
let actors_on_worker: Vec<_> = stats
.actor_details
.iter()
.filter(|a| a.worker_id == id)
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok(
"worker",
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
"tick_phases": phase_breakdown,
"actor_details": actors_on_worker,
}),
)
}
}
pub struct ActorsCommand;
impl CommandHandler for ActorsCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "actors",
description: "List actors with optional sorting, limit, and worker filter",
usage: "actors [--sort mailbox|worker|address] [--limit N] [--worker W]",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.enriched_stats();
let mut actors = stats.actor_details.clone();
let sort_by = arg_str(args, "sort").unwrap_or("mailbox");
let limit = arg_usize(args, "limit").unwrap_or(usize::MAX);
let worker_filter = arg_usize(args, "worker");
if let Some(wid) = worker_filter {
actors.retain(|a| a.worker_id == wid);
}
match sort_by {
"mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)),
"worker" => actors.sort_by_key(|a| a.worker_id),
"address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)),
other => {
return CommandResponse::err(
"actors",
format!("unknown sort field `{other}` — use mailbox|worker|address"),
)
}
}
actors.truncate(limit);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok(
"actors",
serde_json::json!({
"total": stats.actor_details.len(),
"returned": rows.len(),
"sort": sort_by,
"actors": rows,
}),
)
}
}
pub struct ActorCommand;
impl CommandHandler for ActorCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "actor",
description: "Find actor(s) whose address starts with the given hex prefix",
usage: "actor <hex_prefix>",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let prefix = match arg_str(args, "prefix") {
Some(p) => p,
None => return CommandResponse::err("actor", "usage: actor <hex_prefix>"),
};
let stats = ctx.enriched_stats();
let matches: Vec<_> = stats
.actor_details
.iter()
.filter(|a| full_hex(&a.address).starts_with(prefix))
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok(
"actor",
serde_json::json!({
"prefix": prefix,
"matches": matches.len(),
"actors": matches,
}),
)
}
}
pub struct HotCommand;
impl CommandHandler for HotCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "hot",
description: "Top N actors by mailbox depth (default 10)",
usage: "hot [N]",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let n = arg_usize(args, "n").unwrap_or(10);
let stats = ctx.enriched_stats();
let mut actors = stats.actor_details.clone();
actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth));
actors.truncate(n);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok("hot", rows)
}
}
pub struct PhasesCommand;
impl CommandHandler for PhasesCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "phases",
description: "Tick-phase time breakdown (all workers or one)",
usage: "phases [worker_id]",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.stats();
let worker_filter = arg_usize(args, "worker");
let phase_names = [
"spawn_drain",
"transfer_drain",
"tick_all",
"spawn_drain_2",
"pending_local",
"stats_publish",
];
let mut results = Vec::new();
for (i, timings) in stats.tick_timings.iter().enumerate() {
if let Some(wid) = worker_filter {
if i != wid {
continue;
}
}
let breakdown = compute_phase_breakdown(timings);
results.push(serde_json::json!({
"worker_id": i,
"ticks_sampled": timings.len(),
"phases": breakdown,
"phase_names": phase_names,
}));
}
CommandResponse::ok("phases", results)
}
}
pub struct DiffCommand;
impl CommandHandler for DiffCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "diff",
description: "Collect two snapshots N seconds apart, report deltas and rates",
usage: "diff <seconds>",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let secs = match arg_f64(args, "seconds") {
Some(s) if s > 0.0 && s <= 30.0 => s,
Some(_) => return CommandResponse::err("diff", "seconds must be between 0 and 30"),
None => return CommandResponse::err("diff", "usage: diff <seconds>"),
};
let before = ctx.enriched_stats();
let t0 = Instant::now();
std::thread::sleep(Duration::from_secs_f64(secs));
let after = ctx.enriched_stats();
let elapsed = t0.elapsed().as_secs_f64();
let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum();
let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum();
let delta_msgs = msgs_after.saturating_sub(msgs_before);
let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum();
let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum();
let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum();
let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum();
let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum();
let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum();
let per_worker: Vec<_> = after
.workers
.iter()
.enumerate()
.map(|(i, w)| {
let prev = before.workers.get(i);
let d = prev
.map(|p| w.messages_processed.saturating_sub(p.messages_processed))
.unwrap_or(0);
serde_json::json!({
"worker_id": i,
"delta_messages": d,
"msg_per_sec": d as f64 / elapsed,
"actors_before": prev.map(|p| p.num_actors).unwrap_or(0),
"actors_after": w.num_actors,
"mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0),
"mailbox_after": w.mailbox_depth,
})
})
.collect();
CommandResponse::ok(
"diff",
serde_json::json!({
"elapsed_s": elapsed,
"actors_before": before.actor_details.len(),
"actors_after": after.actor_details.len(),
"delta_messages": delta_msgs,
"msg_per_sec": delta_msgs as f64 / elapsed,
"delta_local_sends": local_after.saturating_sub(local_before),
"delta_cross_sends": cross_after.saturating_sub(cross_before),
"mailbox_before": mailbox_before,
"mailbox_after": mailbox_after,
"per_worker": per_worker,
}),
)
}
}
// ─── Write Commands ──────────────────────────────────────────────────────────
pub struct ShutdownCommand;
impl CommandHandler for ShutdownCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "shutdown",
description: "Signal the runtime to shut down gracefully",
usage: "shutdown",
is_write: true,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
ctx.runtime.shutdown();
CommandResponse::ok(
"shutdown",
serde_json::json!({"status": "shutdown signaled"}),
)
}
}

242
crates/command/src/lib.rs Normal file
View file

@ -0,0 +1,242 @@
//! Frontend-agnostic command dispatch for swactor runtimes.
//!
//! Provides [`CommandRouter`] that maps command names to [`CommandHandler`]
//! implementations, with built-in commands for runtime inspection and management.
//!
//! # Architecture
//!
//! ```text
//! Frontend (REPL, REST, TUI, WebSocket)
//! │
//! ▼
//! CommandRouter::dispatch(CommandRequest, CommandContext)
//! │
//! ├── built-in handlers (overview, workers, actors, …)
//! └── custom handlers (user-registered)
//! ```
pub mod builtins;
mod parse;
pub use parse::{from_query_params, parse_line};
use std::collections::HashMap;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use swactor::runtime::Runtime;
use swactor::stats::RuntimeStats;
// ─── Core Types ──────────────────────────────────────────────────────────────
/// A command request.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandRequest {
pub command: String,
pub args: HashMap<String, serde_json::Value>,
}
/// A command response. Always JSON-serializable.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandResponse {
pub ok: bool,
pub command: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub data: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
impl CommandResponse {
pub fn ok(command: &str, data: impl Serialize) -> Self {
Self {
ok: true,
command: command.to_string(),
data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)),
error: None,
}
}
pub fn err(command: &str, msg: impl Into<String>) -> Self {
Self {
ok: false,
command: command.to_string(),
data: None,
error: Some(msg.into()),
}
}
/// Serialize to a single JSON line (for REPL/wire protocol).
pub fn to_json_line(&self) -> String {
serde_json::to_string(self).unwrap_or_else(|e| {
format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#)
})
}
}
// ─── Handler Trait ───────────────────────────────────────────────────────────
/// Metadata about a command, used for help text and validation.
pub struct CommandMeta {
pub name: &'static str,
pub description: &'static str,
pub usage: &'static str,
pub is_write: bool,
}
/// A command handler. Implementations are stateless — all state
/// comes through [`CommandContext`].
pub trait CommandHandler: Send + Sync {
fn meta(&self) -> CommandMeta;
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse;
}
// ─── Stats Enrichment ────────────────────────────────────────────────────────
/// Enriches [`RuntimeStats`] with per-actor detail.
///
/// Implement this on your stats collector so command handlers can access
/// enriched data without depending on the dashboard crate.
pub trait StatsEnricher: Send + Sync {
fn enrich(&self, stats: &mut RuntimeStats);
}
// ─── Context ─────────────────────────────────────────────────────────────────
/// Context available to command handlers.
pub struct CommandContext {
pub runtime: Arc<Runtime>,
pub enricher: Option<Arc<dyn StatsEnricher>>,
}
impl CommandContext {
pub fn new(runtime: Arc<Runtime>) -> Self {
Self {
runtime,
enricher: None,
}
}
pub fn with_enricher(
runtime: Arc<Runtime>,
enricher: Arc<dyn StatsEnricher>,
) -> Self {
Self {
runtime,
enricher: Some(enricher),
}
}
/// Raw runtime stats (no enrichment).
pub fn stats(&self) -> RuntimeStats {
self.runtime.stats()
}
/// Runtime stats enriched with per-actor detail (if an enricher is set).
pub fn enriched_stats(&self) -> RuntimeStats {
let mut s = self.runtime.stats();
if let Some(e) = &self.enricher {
e.enrich(&mut s);
}
s
}
}
// ─── Router ──────────────────────────────────────────────────────────────────
/// Central command dispatch.
pub struct CommandRouter {
handlers: HashMap<String, Box<dyn CommandHandler>>,
}
impl CommandRouter {
pub fn new() -> Self {
Self {
handlers: HashMap::new(),
}
}
/// Create a router with all built-in commands registered.
pub fn with_builtins() -> Self {
let mut router = Self::new();
router.register(Box::new(builtins::OverviewCommand));
router.register(Box::new(builtins::WorkersCommand));
router.register(Box::new(builtins::WorkerCommand));
router.register(Box::new(builtins::ActorsCommand));
router.register(Box::new(builtins::ActorCommand));
router.register(Box::new(builtins::HotCommand));
router.register(Box::new(builtins::PhasesCommand));
router.register(Box::new(builtins::DiffCommand));
router.register(Box::new(builtins::ShutdownCommand));
router
}
/// Register a custom command handler.
pub fn register(&mut self, handler: Box<dyn CommandHandler>) {
let name = handler.meta().name.to_string();
self.handlers.insert(name, handler);
}
/// Dispatch a command request.
///
/// The `help` command is handled directly by the router (it needs
/// access to all registered handlers).
pub fn dispatch(
&self,
req: &CommandRequest,
ctx: &CommandContext,
) -> CommandResponse {
if req.command == "help" {
return self.cmd_help();
}
match self.handlers.get(&req.command) {
Some(handler) => handler.handle(&req.args, ctx),
None => CommandResponse::err(
&req.command,
format!("unknown command `{}` — try `help`", req.command),
),
}
}
fn cmd_help(&self) -> CommandResponse {
let mut commands: Vec<serde_json::Value> = self
.handlers
.values()
.map(|h| {
let m = h.meta();
serde_json::json!({
"name": m.name,
"usage": m.usage,
"description": m.description,
"is_write": m.is_write,
})
})
.collect();
// Add help itself
commands.push(serde_json::json!({
"name": "help",
"usage": "help",
"description": "List all available commands",
"is_write": false,
}));
commands.sort_by(|a, b| {
a["name"]
.as_str()
.unwrap_or("")
.cmp(b["name"].as_str().unwrap_or(""))
});
CommandResponse::ok("help", serde_json::json!({ "commands": commands }))
}
/// List names of all registered commands (sorted).
pub fn command_names(&self) -> Vec<&str> {
let mut names: Vec<_> = self.handlers.keys().map(|s| s.as_str()).collect();
names.push("help");
names.sort();
names
}
}

View file

@ -0,0 +1,90 @@
//! Input parsers for REPL lines and HTTP query parameters.
use std::collections::HashMap;
use crate::CommandRequest;
/// Parse a REPL text line into a [`CommandRequest`].
///
/// Handles `--flag value` pairs and maps positional arguments to
/// command-specific named parameters.
///
/// # Examples
///
/// ```text
/// "overview" → { command: "overview", args: {} }
/// "worker 3" → { command: "worker", args: { "id": "3" } }
/// "actors --sort mailbox" → { command: "actors", args: { "sort": "mailbox" } }
/// "hot 5" → { command: "hot", args: { "n": "5" } }
/// ```
pub fn parse_line(line: &str) -> CommandRequest {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.is_empty() {
return CommandRequest {
command: "help".to_string(),
args: HashMap::new(),
};
}
let command = parts[0].to_string();
let rest = &parts[1..];
let mut args = HashMap::new();
let mut i = 0;
let mut positional = 0;
while i < rest.len() {
if let Some(key) = rest[i].strip_prefix("--") {
if i + 1 < rest.len() && !rest[i + 1].starts_with("--") {
args.insert(
key.to_string(),
serde_json::Value::String(rest[i + 1].to_string()),
);
i += 2;
} else {
args.insert(key.to_string(), serde_json::Value::Bool(true));
i += 1;
}
} else {
let name = positional_arg_name(&command, positional);
if !name.is_empty() {
args.insert(
name.to_string(),
serde_json::Value::String(rest[i].to_string()),
);
}
positional += 1;
i += 1;
}
}
CommandRequest { command, args }
}
/// Convert HTTP query parameters to a [`CommandRequest`].
///
/// The `cmd` parameter becomes the command name; all other parameters
/// become string-valued arguments.
pub fn from_query_params(params: &HashMap<String, String>) -> CommandRequest {
let command = params
.get("cmd")
.cloned()
.unwrap_or_else(|| "help".into());
let args: HashMap<String, serde_json::Value> = params
.iter()
.filter(|(k, _)| *k != "cmd")
.map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone())))
.collect();
CommandRequest { command, args }
}
/// Map positional argument index to the named parameter for each command.
fn positional_arg_name(command: &str, position: usize) -> &'static str {
match (command, position) {
("worker", 0) => "id",
("actor", 0) => "prefix",
("hot", 0) => "n",
("phases", 0) => "worker",
("diff", 0) => "seconds",
_ => "",
}
}

View file

@ -0,0 +1,367 @@
//! Behavioral tests for the swactor-command crate.
//!
//! Tests exercise the full dispatch path: parse → route → handle → response.
use std::collections::HashMap;
use std::sync::Arc;
use swactor::actor::ActorInterface;
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
use swactor_command::{
from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter,
};
// ── Test Helpers ─────────────────────────────────────────────────────────────
fn single_thread_config() -> RuntimeConfig {
RuntimeConfig {
num_threads: 1,
..RuntimeConfig::default()
}
}
fn make_router_and_ctx() -> (CommandRouter, CommandContext) {
let rt = Arc::new(Runtime::new(single_thread_config()));
let router = CommandRouter::with_builtins();
let ctx = CommandContext::new(rt);
(router, ctx)
}
fn dispatch_text(router: &CommandRouter, ctx: &CommandContext, line: &str) -> CommandResponse {
let req = parse_line(line);
let resp = router.dispatch(&req, ctx);
// Verify JSON round-trip works
let json = resp.to_json_line();
serde_json::from_str::<CommandResponse>(&json)
.expect("response should be valid JSON")
}
/// A no-op actor for spawning into the runtime.
struct DummyActor;
#[derive(Clone)]
struct DummyMsg;
impl ActorInterface for DummyActor {
type Incoming = DummyMsg;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: DummyMsg) {}
}
// ── Tests ────────────────────────────────────────────────────────────────────
/// Given a router with builtins,
/// when "help" is dispatched,
/// then the response lists all registered commands.
#[test]
fn help_lists_all_registered_commands() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "help");
assert!(resp.ok, "help should succeed");
assert_eq!(resp.command, "help");
let data = resp.data.unwrap();
let commands = data["commands"].as_array().unwrap();
// Should have all builtins + help itself
let names: Vec<&str> = commands
.iter()
.map(|c| c["name"].as_str().unwrap())
.collect();
assert!(names.contains(&"overview"), "should list overview");
assert!(names.contains(&"workers"), "should list workers");
assert!(names.contains(&"worker"), "should list worker");
assert!(names.contains(&"actors"), "should list actors");
assert!(names.contains(&"hot"), "should list hot");
assert!(names.contains(&"phases"), "should list phases");
assert!(names.contains(&"diff"), "should list diff");
assert!(names.contains(&"shutdown"), "should list shutdown");
assert!(names.contains(&"help"), "should list help itself");
// Should be sorted
let mut sorted = names.clone();
sorted.sort();
assert_eq!(names, sorted, "commands should be sorted alphabetically");
}
/// Given a router,
/// when an unknown command is dispatched,
/// then the response indicates failure with a helpful message.
#[test]
fn unknown_command_returns_error() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "nonexistent");
assert!(!resp.ok, "unknown command should fail");
assert_eq!(resp.command, "nonexistent");
let err = resp.error.unwrap();
assert!(
err.contains("unknown command") && err.contains("help"),
"error should mention 'unknown command' and suggest 'help', got: {err}"
);
}
/// Given a runtime with no actors,
/// when "overview" is dispatched,
/// then the response contains expected summary fields with zero counts.
#[test]
fn overview_returns_summary_fields() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "overview");
assert!(resp.ok);
assert_eq!(resp.command, "overview");
let data = resp.data.unwrap();
assert_eq!(data["workers"], 1, "single-threaded = 1 worker");
assert_eq!(data["actors"], 0, "no actors spawned");
assert_eq!(data["total_messages_processed"], 0);
assert_eq!(data["total_panics"], 0);
assert!(data["sends"].is_object(), "sends should be an object");
}
/// Given a runtime with spawned actors,
/// when "workers" is dispatched,
/// then the response contains per-worker stats.
#[test]
fn workers_returns_per_worker_info() {
let rt = Arc::new(Runtime::new(single_thread_config()));
// Spawn some actors
rt.spawn(DummyActor).unwrap();
rt.spawn(DummyActor).unwrap();
rt.tick();
let router = CommandRouter::with_builtins();
let ctx = CommandContext::new(rt);
let resp = dispatch_text(&router, &ctx, "workers");
assert!(resp.ok);
let data = resp.data.unwrap();
let workers = data.as_array().unwrap();
assert_eq!(workers.len(), 1, "single-threaded has 1 worker");
assert_eq!(workers[0]["id"], 0);
assert_eq!(workers[0]["actors"], 2, "2 actors spawned on worker 0");
}
/// Given "worker 0" with a valid ID,
/// when dispatched,
/// then the response includes worker detail and tick phase info.
#[test]
fn worker_command_with_valid_id() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "worker 0");
assert!(resp.ok);
assert_eq!(resp.command, "worker");
let data = resp.data.unwrap();
assert_eq!(data["id"], 0);
assert!(data["tick_phases"].is_object(), "should include phase breakdown");
}
/// Given "worker 99",
/// when dispatched,
/// then the response is an error (worker not found).
#[test]
fn worker_command_invalid_id_returns_error() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "worker 99");
assert!(!resp.ok);
assert!(resp.error.unwrap().contains("not found"));
}
/// Given "worker" with no ID,
/// when dispatched,
/// then the response is a usage error.
#[test]
fn worker_command_missing_id_returns_usage() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "worker");
assert!(!resp.ok);
assert!(resp.error.unwrap().contains("usage"));
}
/// Given "phases",
/// when dispatched,
/// then the response includes phase breakdown per worker.
#[test]
fn phases_command_returns_breakdown() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "phases");
assert!(resp.ok);
let data = resp.data.unwrap();
let phases = data.as_array().unwrap();
assert_eq!(phases.len(), 1, "single-threaded = 1 worker");
assert_eq!(phases[0]["worker_id"], 0);
}
/// Given a runtime, when "shutdown" is dispatched,
/// then the response indicates success.
#[test]
fn shutdown_command_signals_runtime() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "shutdown");
assert!(resp.ok);
assert_eq!(resp.command, "shutdown");
let data = resp.data.unwrap();
assert_eq!(data["status"], "shutdown signaled");
}
// ── REPL Parser Tests ────────────────────────────────────────────────────────
/// Given a simple command with no args,
/// when parsed,
/// then the command name is extracted correctly.
#[test]
fn parse_line_simple_command() {
let req = parse_line("overview");
assert_eq!(req.command, "overview");
assert!(req.args.is_empty());
}
/// Given a command with positional args,
/// when parsed,
/// then positional args are mapped to named parameters.
#[test]
fn parse_line_positional_args() {
let req = parse_line("worker 3");
assert_eq!(req.command, "worker");
assert_eq!(req.args["id"], "3");
let req = parse_line("hot 5");
assert_eq!(req.command, "hot");
assert_eq!(req.args["n"], "5");
let req = parse_line("diff 2.5");
assert_eq!(req.command, "diff");
assert_eq!(req.args["seconds"], "2.5");
let req = parse_line("actor a1b2");
assert_eq!(req.command, "actor");
assert_eq!(req.args["prefix"], "a1b2");
}
/// Given a command with --flag value pairs,
/// when parsed,
/// then flags are mapped to named args.
#[test]
fn parse_line_flags() {
let req = parse_line("actors --sort mailbox --limit 5");
assert_eq!(req.command, "actors");
assert_eq!(req.args["sort"], "mailbox");
assert_eq!(req.args["limit"], "5");
}
/// Given a command with mixed positional and flag args,
/// when parsed,
/// then both are captured correctly.
#[test]
fn parse_line_mixed_args() {
let req = parse_line("actors --sort worker --worker 2 --limit 10");
assert_eq!(req.command, "actors");
assert_eq!(req.args["sort"], "worker");
assert_eq!(req.args["worker"], "2");
assert_eq!(req.args["limit"], "10");
}
/// Given empty input,
/// when parsed,
/// then default to "help".
#[test]
fn parse_line_empty_defaults_to_help() {
let req = parse_line("");
assert_eq!(req.command, "help");
}
// ── REST Adapter Tests ───────────────────────────────────────────────────────
/// Given query params with cmd and other params,
/// when converted,
/// then cmd becomes the command and others become args.
#[test]
fn from_query_params_extracts_cmd() {
let mut params = HashMap::new();
params.insert("cmd".to_string(), "actor".to_string());
params.insert("prefix".to_string(), "a1b2".to_string());
let req = from_query_params(&params);
assert_eq!(req.command, "actor");
assert_eq!(req.args["prefix"], "a1b2");
assert!(!req.args.contains_key("cmd"), "cmd should not be in args");
}
/// Given query params with no cmd,
/// when converted,
/// then default to "help".
#[test]
fn from_query_params_defaults_to_help() {
let params = HashMap::new();
let req = from_query_params(&params);
assert_eq!(req.command, "help");
}
// ── Custom Handler Test ──────────────────────────────────────────────────────
/// Given a custom command handler registered on the router,
/// when that command is dispatched,
/// then the custom handler runs and returns its response.
#[test]
fn custom_command_handler() {
struct PingCommand;
impl swactor_command::CommandHandler for PingCommand {
fn meta(&self) -> swactor_command::CommandMeta {
swactor_command::CommandMeta {
name: "ping",
description: "Respond with pong",
usage: "ping",
is_write: false,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
_ctx: &CommandContext,
) -> CommandResponse {
CommandResponse::ok("ping", serde_json::json!({"reply": "pong"}))
}
}
let rt = Arc::new(Runtime::new(single_thread_config()));
let mut router = CommandRouter::with_builtins();
router.register(Box::new(PingCommand));
let ctx = CommandContext::new(rt);
let resp = dispatch_text(&router, &ctx, "ping");
assert!(resp.ok);
assert_eq!(resp.data.unwrap()["reply"], "pong");
// Should also appear in help
let help = dispatch_text(&router, &ctx, "help");
let commands = help.data.unwrap()["commands"].as_array().unwrap().clone();
let names: Vec<&str> = commands.iter().map(|c| c["name"].as_str().unwrap()).collect();
assert!(names.contains(&"ping"), "custom command should appear in help");
}
/// Given a JSON-serialized CommandResponse,
/// when deserialized,
/// then ok/false fields, data, and error are preserved.
#[test]
fn response_json_roundtrip() {
let ok_resp = CommandResponse::ok("test", serde_json::json!({"key": "value"}));
let json = ok_resp.to_json_line();
let parsed: CommandResponse = serde_json::from_str(&json).unwrap();
assert!(parsed.ok);
assert_eq!(parsed.command, "test");
assert_eq!(parsed.data.unwrap()["key"], "value");
assert!(parsed.error.is_none());
let err_resp = CommandResponse::err("bad", "something went wrong");
let json = err_resp.to_json_line();
let parsed: CommandResponse = serde_json::from_str(&json).unwrap();
assert!(!parsed.ok);
assert_eq!(parsed.command, "bad");
assert!(parsed.data.is_none());
assert_eq!(parsed.error.unwrap(), "something went wrong");
}

View file

@ -7,4 +7,5 @@ pub mod swim;
pub mod kademlia; pub mod kademlia;
pub mod cache; pub mod cache;
pub mod node; pub mod node;
pub mod registry;
pub mod snapshot; pub mod snapshot;

View file

@ -12,6 +12,10 @@ use crate::crypto::Keypair;
use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard}; use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard};
use crate::kademlia::repair::{RepairQueue, RepublishTracker}; use crate::kademlia::repair::{RepairQueue, RepublishTracker};
use crate::kademlia::routing_table::RoutingTable; use crate::kademlia::routing_table::RoutingTable;
use crate::registry::{
pack_combined_piggyback, unpack_combined_piggyback, ClusterRegistry, RegistryConfig,
RegistryEvent,
};
use crate::swim::node::{NodeAction, SwimNode}; use crate::swim::node::{NodeAction, SwimNode};
use crate::swim::probe::SwimConfig; use crate::swim::probe::SwimConfig;
use crate::types::{MemberState, NodeId, NodeRecord}; use crate::types::{MemberState, NodeId, NodeRecord};
@ -22,6 +26,7 @@ pub struct DistributedNodeConfig {
pub swim: SwimConfig, pub swim: SwimConfig,
pub cache_capacity: usize, pub cache_capacity: usize,
pub republish_interval: u64, pub republish_interval: u64,
pub registry: RegistryConfig,
} }
impl Default for DistributedNodeConfig { impl Default for DistributedNodeConfig {
@ -31,6 +36,7 @@ impl Default for DistributedNodeConfig {
swim: SwimConfig::default(), swim: SwimConfig::default(),
cache_capacity: 10_000, cache_capacity: 10_000,
republish_interval: 1000, republish_interval: 1000,
registry: RegistryConfig::default(),
} }
} }
} }
@ -47,6 +53,7 @@ pub struct DistributedNode {
cache: LocationCache, cache: LocationCache,
repair_queue: RepairQueue, repair_queue: RepairQueue,
republish: RepublishTracker, republish: RepublishTracker,
registry: ClusterRegistry,
tick_count: u64, tick_count: u64,
} }
@ -67,6 +74,7 @@ impl DistributedNode {
cache: LocationCache::new(config.cache_capacity), cache: LocationCache::new(config.cache_capacity),
repair_queue: RepairQueue::new(), repair_queue: RepairQueue::new(),
republish: RepublishTracker::new(config.republish_interval), republish: RepublishTracker::new(config.republish_interval),
registry: ClusterRegistry::new(config.registry),
tick_count: 0, tick_count: 0,
keypair, keypair,
} }
@ -149,23 +157,32 @@ impl DistributedNode {
// re-sign and re-STORE these entries. // re-sign and re-STORE these entries.
} }
actions // Registry GC
self.registry.gc_tick();
// Wrap outgoing piggyback with registry entries
self.inject_registry_piggyback(actions)
} }
// ─── SWIM message handling (delegate to SwimNode) ─────────────────── // ─── SWIM message handling (delegate to SwimNode) ───────────────────
pub fn handle_ping(&mut self, from: NodeId, from_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> { pub fn handle_ping(&mut self, from: NodeId, from_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
let actions = self.swim.handle_ping(from, from_addr, sequence, piggyback); let membership_bytes = self.extract_registry_piggyback(piggyback);
let actions = self.swim.handle_ping(from, from_addr, sequence, &membership_bytes);
self.maybe_update_routing_table(from, from_addr); self.maybe_update_routing_table(from, from_addr);
actions self.inject_registry_piggyback(actions)
} }
pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> { pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
self.swim.handle_ack(from, sequence, piggyback) let membership_bytes = self.extract_registry_piggyback(piggyback);
let actions = self.swim.handle_ack(from, sequence, &membership_bytes);
self.inject_registry_piggyback(actions)
} }
pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, target_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> { pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, target_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
self.swim.handle_ping_req(from, target, target_addr, sequence, piggyback) let membership_bytes = self.extract_registry_piggyback(piggyback);
let actions = self.swim.handle_ping_req(from, target, target_addr, sequence, &membership_bytes);
self.inject_registry_piggyback(actions)
} }
pub fn handle_join_request(&mut self, from: NodeId, from_addr: SocketAddr) -> Vec<NodeAction> { pub fn handle_join_request(&mut self, from: NodeId, from_addr: SocketAddr) -> Vec<NodeAction> {
@ -233,6 +250,33 @@ impl DistributedNode {
self.cache.invalidate(actor_addr); self.cache.invalidate(actor_addr);
} }
// ─── Registry (name → actor mapping) ──────────────────────────────
/// Register a human-readable name for an actor on this node.
pub fn register_name(&mut self, name: String, actor_addr: ActorAddress) {
self.registry.register(name, actor_addr, self.node_id(), self.cluster_size());
}
/// Unregister a name (creates a tombstone).
pub fn unregister_name(&mut self, name: &str) {
self.registry.unregister(name, self.node_id(), self.cluster_size());
}
/// Resolve a name to its current (ActorAddress, NodeId).
pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
self.registry.resolve(name)
}
/// Drain registry events (Registered / Unregistered).
pub fn registry_events(&mut self) -> Vec<RegistryEvent> {
self.registry.drain_events()
}
/// Read-only access to the registry.
pub fn registry(&self) -> &ClusterRegistry {
&self.registry
}
// ─── Accessors ────────────────────────────────────────────────────── // ─── Accessors ──────────────────────────────────────────────────────
pub fn routing_table(&self) -> &RoutingTable { pub fn routing_table(&self) -> &RoutingTable {
@ -277,12 +321,51 @@ impl DistributedNode {
self.routing_table.remove(&node_id); self.routing_table.remove(&node_id);
self.cache.invalidate_node(&node_id); self.cache.invalidate_node(&node_id);
self.repair_queue.on_node_death(&node_id, &mut self.directory); self.repair_queue.on_node_death(&node_id, &mut self.directory);
self.registry.tombstone_node(node_id, self.cluster_size());
} }
MemberState::Suspect => { MemberState::Suspect => {
// Keep in routing table but could downprioritize // Keep in routing table but could downprioritize
} }
} }
} }
fn cluster_size(&self) -> usize {
self.swim.members().alive_count() + 1 // +1 for self
}
/// Post-process outgoing actions: wrap each piggyback with registry entries.
fn inject_registry_piggyback(&mut self, actions: Vec<NodeAction>) -> Vec<NodeAction> {
actions
.into_iter()
.map(|action| match action {
NodeAction::SendPing { to, to_addr, sequence, piggyback } => {
let registry_entries = self.registry.take_pending(8);
let combined = pack_combined_piggyback(piggyback, registry_entries);
NodeAction::SendPing { to, to_addr, sequence, piggyback: combined }
}
NodeAction::SendAck { to, to_addr, sequence, piggyback } => {
let registry_entries = self.registry.take_pending(8);
let combined = pack_combined_piggyback(piggyback, registry_entries);
NodeAction::SendAck { to, to_addr, sequence, piggyback: combined }
}
NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback } => {
let registry_entries = self.registry.take_pending(8);
let combined = pack_combined_piggyback(piggyback, registry_entries);
NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback: combined }
}
other => other,
})
.collect()
}
/// Extract registry entries from incoming piggyback, merge them, return membership-only bytes.
fn extract_registry_piggyback(&mut self, bytes: &[u8]) -> Vec<u8> {
let (membership_bytes, registry_entries) = unpack_combined_piggyback(bytes);
if !registry_entries.is_empty() {
self.registry.merge_batch(registry_entries, self.cluster_size());
}
membership_bytes
}
} }
/// Result of resolving an actor's location. /// Result of resolving an actor's location.

View file

@ -0,0 +1,374 @@
//! Cluster Registry — gossip-propagated naming via LWW-Register CRDT.
//!
//! Maps human-readable names to `(ActorAddress, NodeId)` pairs, propagated
//! through SWIM gossip piggyback. Uses last-writer-wins semantics with
//! tie-breaking on (timestamp, generation, node_id).
use std::collections::{HashMap, VecDeque};
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use crate::types::NodeId;
// ─── Configuration ──────────────────────────────────────────────────────────
/// Configuration for the cluster registry.
pub struct RegistryConfig {
/// Maximum number of events to buffer before dropping old ones.
pub max_events: usize,
/// How long (in ticks) a tombstone is retained before GC.
pub tombstone_ttl: u64,
/// How often (in ticks) to run garbage collection.
pub gc_interval: u64,
/// Dissemination multiplier (Λ) — same role as in SWIM dissemination.
pub dissemination_lambda: usize,
}
impl Default for RegistryConfig {
fn default() -> Self {
Self {
max_events: 256,
tombstone_ttl: 3600,
gc_interval: 1000,
dissemination_lambda: 3,
}
}
}
// ─── Wire types ─────────────────────────────────────────────────────────────
/// A single registry entry — the unit of replication.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RegistryEntry {
pub name: String,
pub actor_addr: ActorAddress,
pub node_id: NodeId,
/// Logical timestamp (monotonically increasing per-registry).
pub timestamp: u64,
/// Generation counter for the same name (disambiguates re-registrations).
pub generation: u64,
/// If true, this entry is a tombstone (name was unregistered).
pub tombstone: bool,
}
/// Combined piggyback payload: membership bytes + registry entries.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PiggybackPayload {
/// Raw SWIM membership piggyback bytes (opaque to registry).
pub membership: Vec<u8>,
/// Registry entries to disseminate.
pub registry: Vec<RegistryEntry>,
}
// ─── Events ─────────────────────────────────────────────────────────────────
/// Events emitted when the registry changes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RegistryEvent {
Registered {
name: String,
actor_addr: ActorAddress,
node_id: NodeId,
},
Unregistered {
name: String,
previous_addr: ActorAddress,
},
}
// ─── Dissemination entry ────────────────────────────────────────────────────
#[derive(Debug, Clone)]
struct DisseminationEntry {
entry: RegistryEntry,
remaining: usize,
}
// ─── ClusterRegistry ────────────────────────────────────────────────────────
/// CRDT-based cluster registry with LWW semantics and gossip dissemination.
pub struct ClusterRegistry {
/// Current state: name → latest entry.
entries: HashMap<String, RegistryEntry>,
/// Pending entries to disseminate via piggyback.
dissemination: Vec<DisseminationEntry>,
/// Monotonic logical clock for this node's writes.
clock: u64,
/// Buffered events for consumers.
events: VecDeque<RegistryEvent>,
config: RegistryConfig,
tick_count: u64,
}
impl ClusterRegistry {
pub fn new(config: RegistryConfig) -> Self {
Self {
entries: HashMap::new(),
dissemination: Vec::new(),
clock: 0,
events: VecDeque::new(),
config,
tick_count: 0,
}
}
/// Register a name → actor binding from the local node.
pub fn register(&mut self, name: String, actor_addr: ActorAddress, node_id: NodeId, cluster_size: usize) {
self.clock += 1;
let generation = self.next_generation(&name);
let entry = RegistryEntry {
name,
actor_addr,
node_id,
timestamp: self.clock,
generation,
tombstone: false,
};
self.merge_and_enqueue(entry, cluster_size);
}
/// Unregister a name (create a tombstone).
pub fn unregister(&mut self, name: &str, node_id: NodeId, cluster_size: usize) {
self.clock += 1;
let generation = self.next_generation(name);
// Use the existing actor_addr if present, otherwise a zero address.
let actor_addr = self.entries
.get(name)
.map(|e| e.actor_addr)
.unwrap_or(ActorAddress([0; 32]));
let entry = RegistryEntry {
name: name.to_string(),
actor_addr,
node_id,
timestamp: self.clock,
generation,
tombstone: true,
};
self.merge_and_enqueue(entry, cluster_size);
}
/// Resolve a name to its current (ActorAddress, NodeId), or None if
/// not registered or tombstoned.
pub fn resolve(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
self.entries.get(name).and_then(|e| {
if e.tombstone {
None
} else {
Some((e.actor_addr, e.node_id))
}
})
}
/// Merge a single remote entry. Returns true if state changed.
pub fn merge(&mut self, remote: RegistryEntry) -> bool {
if let Some(existing) = self.entries.get(&remote.name) {
if !lww_wins(&remote, existing) {
return false;
}
}
let changed = match self.entries.get(&remote.name) {
Some(existing) => existing != &remote,
None => true,
};
if changed {
self.emit_event(&remote);
// Advance clock to stay ahead of remote timestamps.
if remote.timestamp >= self.clock {
self.clock = remote.timestamp + 1;
}
}
self.entries.insert(remote.name.clone(), remote);
changed
}
/// Merge a batch of entries received from gossip.
/// Changed entries are re-enqueued for further dissemination.
pub fn merge_batch(&mut self, entries: Vec<RegistryEntry>, cluster_size: usize) {
for entry in entries {
if self.merge(entry.clone()) {
self.enqueue(entry, cluster_size);
}
}
}
/// Take pending entries for piggyback, up to `max_count`.
pub fn take_pending(&mut self, max_count: usize) -> Vec<RegistryEntry> {
let count = max_count.min(self.dissemination.len());
let mut result = Vec::with_capacity(count);
for entry in self.dissemination.iter_mut().take(count) {
result.push(entry.entry.clone());
entry.remaining = entry.remaining.saturating_sub(1);
}
// Evict exhausted entries.
self.dissemination.retain(|e| e.remaining > 0);
result
}
/// Tombstone all entries owned by a dead node.
pub fn tombstone_node(&mut self, dead_node_id: NodeId, cluster_size: usize) {
let owned: Vec<String> = self.entries
.iter()
.filter(|(_, e)| e.node_id == dead_node_id && !e.tombstone)
.map(|(name, _)| name.clone())
.collect();
for name in owned {
self.clock += 1;
let generation = self.next_generation(&name);
let actor_addr = self.entries[&name].actor_addr;
let entry = RegistryEntry {
name,
actor_addr,
node_id: dead_node_id,
timestamp: self.clock,
generation,
tombstone: true,
};
self.merge_and_enqueue(entry, cluster_size);
}
}
/// Periodic GC: remove tombstones past TTL with exhausted dissemination budgets.
pub fn gc_tick(&mut self) {
self.tick_count += 1;
if self.tick_count % self.config.gc_interval != 0 {
return;
}
let ttl = self.config.tombstone_ttl;
let clock = self.clock;
// Names still being disseminated — don't GC those.
let pending_names: std::collections::HashSet<String> = self.dissemination
.iter()
.map(|e| e.entry.name.clone())
.collect();
self.entries.retain(|name, entry| {
if entry.tombstone && !pending_names.contains(name) {
// Remove if old enough.
let age = clock.saturating_sub(entry.timestamp);
age < ttl
} else {
true
}
});
}
/// Drain buffered events.
pub fn drain_events(&mut self) -> Vec<RegistryEvent> {
self.events.drain(..).collect()
}
/// Number of registry entries (including tombstones).
pub fn len(&self) -> usize {
self.entries.len()
}
/// Number of tombstones.
pub fn tombstone_count(&self) -> usize {
self.entries.values().filter(|e| e.tombstone).count()
}
/// Iterate all entries (for snapshot).
pub fn entries(&self) -> impl Iterator<Item = &RegistryEntry> {
self.entries.values()
}
// ─── Internal ───────────────────────────────────────────────────────
fn next_generation(&self, name: &str) -> u64 {
self.entries
.get(name)
.map(|e| e.generation + 1)
.unwrap_or(1)
}
fn transmit_budget(&self, cluster_size: usize) -> usize {
let n = cluster_size.max(2) as f64;
let log_n = n.log2().ceil() as usize;
self.config.dissemination_lambda * log_n.max(1)
}
fn enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) {
let budget = self.transmit_budget(cluster_size);
// Replace existing entry for same name if present.
if let Some(existing) = self.dissemination.iter_mut().find(|e| e.entry.name == entry.name) {
existing.entry = entry;
existing.remaining = budget;
return;
}
self.dissemination.push(DisseminationEntry {
entry,
remaining: budget,
});
}
fn merge_and_enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) {
let merged = self.merge(entry.clone());
if merged {
self.enqueue(entry, cluster_size);
}
}
fn emit_event(&mut self, entry: &RegistryEntry) {
let event = if entry.tombstone {
RegistryEvent::Unregistered {
name: entry.name.clone(),
previous_addr: entry.actor_addr,
}
} else {
RegistryEvent::Registered {
name: entry.name.clone(),
actor_addr: entry.actor_addr,
node_id: entry.node_id,
}
};
self.events.push_back(event);
while self.events.len() > self.config.max_events {
self.events.pop_front();
}
}
}
// ─── LWW conflict resolution ───────────────────────────────────────────────
/// Returns true if `incoming` wins over `existing` under LWW rules:
/// higher timestamp > higher generation > higher node_id (byte-level).
fn lww_wins(incoming: &RegistryEntry, existing: &RegistryEntry) -> bool {
if incoming.timestamp != existing.timestamp {
return incoming.timestamp > existing.timestamp;
}
if incoming.generation != existing.generation {
return incoming.generation > existing.generation;
}
incoming.node_id.0 > existing.node_id.0
}
// ─── Piggyback pack/unpack ──────────────────────────────────────────────────
/// Combine membership piggyback bytes and registry entries into a single payload.
pub fn pack_combined_piggyback(membership: Vec<u8>, registry: Vec<RegistryEntry>) -> Vec<u8> {
let payload = PiggybackPayload { membership, registry };
serde_json::to_vec(&payload).unwrap_or_default()
}
/// Split a combined piggyback payload into membership bytes and registry entries.
/// If deserialization fails, treats the entire blob as membership bytes (backwards compat).
pub fn unpack_combined_piggyback(bytes: &[u8]) -> (Vec<u8>, Vec<RegistryEntry>) {
if bytes.is_empty() {
return (Vec::new(), Vec::new());
}
match serde_json::from_slice::<PiggybackPayload>(bytes) {
Ok(payload) => (payload.membership, payload.registry),
Err(_) => (bytes.to_vec(), Vec::new()),
}
}

View file

@ -33,6 +33,15 @@ pub struct CacheEntryInfo {
pub node_id: String, pub node_id: String,
} }
/// Snapshot of a single registry entry.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegistryEntryInfo {
pub name: String,
pub actor_addr: String,
pub node_id: String,
pub tombstone: bool,
}
/// Complete snapshot of a `DistributedNode`'s observable state. /// Complete snapshot of a `DistributedNode`'s observable state.
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DistributionNodeSnapshot { pub struct DistributionNodeSnapshot {
@ -71,6 +80,14 @@ pub struct DistributionNodeSnapshot {
/// Number of entries pending re-replication. /// Number of entries pending re-replication.
pub repair_queue_size: usize, pub repair_queue_size: usize,
// ─── Registry ────────────────────────────────────────────────────
/// Number of entries in the cluster registry (including tombstones).
pub registry_size: usize,
/// Number of tombstoned entries.
pub registry_tombstones: usize,
/// All registry entries.
pub registry_entries: Vec<RegistryEntryInfo>,
// ─── Gossip pairs ──────────────────────────────────────────────── // ─── Gossip pairs ────────────────────────────────────────────────
/// Recent SWIM probe targets (most recent last). /// Recent SWIM probe targets (most recent last).
pub recent_probe_targets: Vec<String>, pub recent_probe_targets: Vec<String>,
@ -136,6 +153,17 @@ impl DistributedNode {
.map(|id| node_id_hex(id)) .map(|id| node_id_hex(id))
.collect(); .collect();
let registry = self.registry();
let registry_entries: Vec<RegistryEntryInfo> = registry
.entries()
.map(|e| RegistryEntryInfo {
name: e.name.clone(),
actor_addr: format!("{}", e.actor_addr),
node_id: node_id_hex(&e.node_id),
tombstone: e.tombstone,
})
.collect();
DistributionNodeSnapshot { DistributionNodeSnapshot {
node_id: node_id_hex(&self.node_id()), node_id: node_id_hex(&self.node_id()),
listen_addr: addr_str(&self.listen_addr()), listen_addr: addr_str(&self.listen_addr()),
@ -150,6 +178,9 @@ impl DistributedNode {
cache_entries, cache_entries,
directory_entry_count: self.directory().entry_count(), directory_entry_count: self.directory().entry_count(),
repair_queue_size: self.repair_queue_len(), repair_queue_size: self.repair_queue_len(),
registry_size: registry.len(),
registry_tombstones: registry.tombstone_count(),
registry_entries,
recent_probe_targets: recent_targets, recent_probe_targets: recent_targets,
} }
} }

View file

@ -9,6 +9,7 @@ use swactor::actor::ActorAddress;
use distribution::crypto::Keypair; use distribution::crypto::Keypair;
use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult}; use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult};
use distribution::swim::node::NodeAction; use distribution::swim::node::NodeAction;
use distribution::registry::RegistryConfig;
use distribution::swim::probe::SwimConfig; use distribution::swim::probe::SwimConfig;
use distribution::types::NodeId; use distribution::types::NodeId;
@ -23,6 +24,7 @@ fn test_config(addr: &str) -> DistributedNodeConfig {
}, },
cache_capacity: 100, cache_capacity: 100,
republish_interval: 50, republish_interval: 50,
registry: RegistryConfig::default(),
} }
} }

View file

@ -0,0 +1,510 @@
//! Behavioral tests for the cluster registry.
//!
//! Tests gossip-propagated naming via LWW-Register CRDT, using the same
//! `deliver_actions` + `test_config` pattern from `node_integration.rs`.
use std::net::SocketAddr;
use swactor::actor::ActorAddress;
use distribution::node::{DistributedNode, DistributedNodeConfig};
use distribution::registry::{ClusterRegistry, RegistryConfig, RegistryEntry, RegistryEvent};
use distribution::swim::node::NodeAction;
use distribution::swim::probe::SwimConfig;
use distribution::types::NodeId;
fn test_config(addr: &str) -> DistributedNodeConfig {
DistributedNodeConfig {
listen_addr: addr.parse().unwrap(),
swim: SwimConfig {
probe_interval: 1,
probe_timeout: 3,
indirect_probes: 1,
suspicion_timeout: 5,
},
cache_capacity: 100,
republish_interval: 50,
registry: RegistryConfig::default(),
}
}
/// Simulate a network round: deliver actions from `sender` to the appropriate
/// `receiver` node. Returns any actions generated by the receiver.
fn deliver_actions(
actions: &[NodeAction],
sender_id: NodeId,
sender_addr: SocketAddr,
nodes: &mut [(NodeId, SocketAddr, &mut DistributedNode)],
) -> Vec<NodeAction> {
let mut responses = Vec::new();
for action in actions {
match action {
NodeAction::SendPing { to, sequence, piggyback, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
responses.extend(node.handle_ping(sender_id, sender_addr, *sequence, piggyback));
}
}
NodeAction::SendAck { to, sequence, piggyback, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
responses.extend(node.handle_ack(sender_id, *sequence, piggyback));
}
}
NodeAction::SendJoinRequest { to_addr } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(_, addr, _)| addr == to_addr) {
responses.extend(node.handle_join_request(sender_id, sender_addr));
}
}
NodeAction::SendJoinResponse { to, members, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
responses.extend(node.handle_join_response(members.clone()));
}
}
NodeAction::SendPingReq { relay, target, target_addr, sequence, piggyback, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == relay) {
responses.extend(node.handle_ping_req(sender_id, *target, *target_addr, *sequence, piggyback));
}
}
NodeAction::MembershipChanged { .. } => {}
}
}
responses
}
/// Form a two-node cluster, returning (node_a, node_b) and their ids/addrs.
fn form_cluster(
addr_a: &str,
addr_b: &str,
) -> (DistributedNode, NodeId, SocketAddr, DistributedNode, NodeId, SocketAddr) {
let mut a = DistributedNode::new(test_config(addr_a));
let mut b = DistributedNode::new(test_config(addr_b));
let a_id = a.node_id();
let a_addr = a.listen_addr();
let b_id = b.node_id();
let b_addr = b.listen_addr();
let actions = b.join(&[a_addr]);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
(a, a_id, a_addr, b, b_id, b_addr)
}
/// Run several gossip rounds between two nodes.
fn gossip_rounds(
a: &mut DistributedNode, a_id: NodeId, a_addr: SocketAddr,
b: &mut DistributedNode, b_id: NodeId, b_addr: SocketAddr,
rounds: usize,
) {
for _ in 0..rounds {
let actions_a = a.tick();
let mut nodes = vec![(b_id, b_addr, &mut *b)];
let responses = deliver_actions(&actions_a, a_id, a_addr, &mut nodes);
let mut nodes = vec![(a_id, a_addr, &mut *a)];
let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes);
let actions_b = b.tick();
let mut nodes = vec![(a_id, a_addr, &mut *a)];
let responses = deliver_actions(&actions_b, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut *b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
}
}
// ─── Test 1: register and resolve ───────────────────────────────────────────
#[test]
fn register_and_resolve() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10001"));
let actor = ActorAddress::new_random();
let node_id = node.node_id();
node.register_name("my-actor".into(), actor);
let result = node.resolve_name("my-actor");
assert_eq!(result, Some((actor, node_id)));
}
// ─── Test 2: unregistered name returns None ─────────────────────────────────
#[test]
fn unregistered_name_returns_none() {
let node = DistributedNode::new(test_config("127.0.0.1:10002"));
assert_eq!(node.resolve_name("nonexistent"), None);
}
// ─── Test 3: unregister tombstones name ─────────────────────────────────────
#[test]
fn unregister_tombstones_name() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10003"));
let actor = ActorAddress::new_random();
node.register_name("service".into(), actor);
assert!(node.resolve_name("service").is_some());
node.unregister_name("service");
assert_eq!(node.resolve_name("service"), None);
}
// ─── Test 4: re-registration updates binding ────────────────────────────────
#[test]
fn re_registration_updates_binding() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10004"));
let actor_a = ActorAddress::new_random();
let actor_b = ActorAddress::new_random();
let node_id = node.node_id();
node.register_name("foo".into(), actor_a);
assert_eq!(node.resolve_name("foo"), Some((actor_a, node_id)));
node.register_name("foo".into(), actor_b);
assert_eq!(node.resolve_name("foo"), Some((actor_b, node_id)));
}
// ─── Test 5: LWW conflict — higher timestamp wins ──────────────────────────
#[test]
fn lww_conflict_higher_timestamp_wins() {
let mut reg = ClusterRegistry::new(RegistryConfig::default());
let addr_old = ActorAddress::new_random();
let addr_new = ActorAddress::new_random();
let node_id = NodeId([1; 32]);
let old_entry = RegistryEntry {
name: "svc".into(),
actor_addr: addr_old,
node_id,
timestamp: 1,
generation: 1,
tombstone: false,
};
let new_entry = RegistryEntry {
name: "svc".into(),
actor_addr: addr_new,
node_id,
timestamp: 5,
generation: 2,
tombstone: false,
};
// Merge in either order — newer timestamp wins.
reg.merge(new_entry.clone());
reg.merge(old_entry.clone());
assert_eq!(reg.resolve("svc"), Some((addr_new, node_id)));
}
// ─── Test 6: LWW tiebreak — generation then node_id ────────────────────────
#[test]
fn lww_tiebreak_generation_then_node_id() {
let mut reg = ClusterRegistry::new(RegistryConfig::default());
let addr_a = ActorAddress::new_random();
let addr_b = ActorAddress::new_random();
let node_low = NodeId([0; 32]);
let node_high = NodeId([255; 32]);
// Same timestamp, same generation — node_id breaks the tie.
let entry_low = RegistryEntry {
name: "x".into(),
actor_addr: addr_a,
node_id: node_low,
timestamp: 10,
generation: 1,
tombstone: false,
};
let entry_high = RegistryEntry {
name: "x".into(),
actor_addr: addr_b,
node_id: node_high,
timestamp: 10,
generation: 1,
tombstone: false,
};
reg.merge(entry_low);
reg.merge(entry_high);
// Higher node_id wins.
assert_eq!(reg.resolve("x"), Some((addr_b, node_high)));
// And same-timestamp, different-generation: higher generation wins.
let mut reg2 = ClusterRegistry::new(RegistryConfig::default());
let entry_gen1 = RegistryEntry {
name: "y".into(),
actor_addr: addr_a,
node_id: node_low,
timestamp: 10,
generation: 1,
tombstone: false,
};
let entry_gen2 = RegistryEntry {
name: "y".into(),
actor_addr: addr_b,
node_id: node_low,
timestamp: 10,
generation: 2,
tombstone: false,
};
reg2.merge(entry_gen1);
reg2.merge(entry_gen2);
assert_eq!(reg2.resolve("y"), Some((addr_b, node_low)));
}
// ─── Test 7: gossip propagates registration ─────────────────────────────────
#[test]
fn gossip_propagates_registration() {
let (mut a, a_id, a_addr, mut b, b_id, b_addr) =
form_cluster("127.0.0.1:10010", "127.0.0.1:10011");
let actor = ActorAddress::new_random();
a.register_name("greeter".into(), actor);
// B doesn't know about "greeter" yet.
assert_eq!(b.resolve_name("greeter"), None);
// Run gossip rounds — registry entries piggyback on SWIM messages.
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
// Now B should resolve "greeter" to A's actor.
assert_eq!(b.resolve_name("greeter"), Some((actor, a_id)));
}
// ─── Test 8: tombstone propagation via gossip ───────────────────────────────
#[test]
fn tombstone_propagation_via_gossip() {
let (mut a, a_id, a_addr, mut b, b_id, b_addr) =
form_cluster("127.0.0.1:10020", "127.0.0.1:10021");
let actor = ActorAddress::new_random();
a.register_name("ephemeral".into(), actor);
// Propagate the registration.
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
assert_eq!(b.resolve_name("ephemeral"), Some((actor, a_id)));
// Now unregister on A.
a.unregister_name("ephemeral");
// Propagate the tombstone.
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
assert_eq!(b.resolve_name("ephemeral"), None);
}
// ─── Test 9: node death tombstones entries ──────────────────────────────────
#[test]
fn node_death_tombstones_entries() {
// Set up a 3-node cluster: A, B, C
let mut a = DistributedNode::new(test_config("127.0.0.1:10030"));
let mut b = DistributedNode::new(test_config("127.0.0.1:10031"));
let mut c = DistributedNode::new(test_config("127.0.0.1:10032"));
let a_id = a.node_id();
let a_addr = a.listen_addr();
let b_id = b.node_id();
let b_addr = b.listen_addr();
let c_id = c.node_id();
let c_addr = c.listen_addr();
// B and C join A.
let actions = b.join(&[a_addr]);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
let actions = c.join(&[a_addr]);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes);
let mut nodes = vec![(c_id, c_addr, &mut c)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
// B registers a name.
let actor = ActorAddress::new_random();
b.register_name("b-service".into(), actor);
// Propagate B's registration to A and C via mesh gossip.
// B only knows A, so first B→A, then A→C carries it.
for _ in 0..5 {
// Each node ticks and delivers to all others.
let actions = b.tick();
let mut nodes = vec![(a_id, a_addr, &mut a), (c_id, c_addr, &mut c)];
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
let actions = a.tick();
let mut nodes = vec![(b_id, b_addr, &mut b), (c_id, c_addr, &mut c)];
let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes);
let actions = c.tick();
let mut nodes = vec![(a_id, a_addr, &mut a), (b_id, b_addr, &mut b)];
let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes);
let mut nodes = vec![(c_id, c_addr, &mut c)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
}
assert_eq!(a.resolve_name("b-service"), Some((actor, b_id)));
assert_eq!(c.resolve_name("b-service"), Some((actor, b_id)));
// B dies — SWIM detects via timeout. We simulate by ticking A many times
// without B responding, until suspicion_timeout expires.
for _ in 0..20 {
let actions = a.tick();
// Don't deliver to B — it's "dead". Only deliver to C.
let mut nodes = vec![(c_id, c_addr, &mut c)];
let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let _ = deliver_actions(&responses, c_id, c_addr, &mut nodes);
}
// After enough ticks, A should declare B dead, which tombstones "b-service".
// Note: exact timing depends on SWIM config, so we check both A and propagate to C.
let a_resolved = a.resolve_name("b-service");
if a_resolved.is_none() {
// A has tombstoned it — propagate to C.
gossip_rounds(&mut a, a_id, a_addr, &mut c, c_id, c_addr, 5);
assert_eq!(c.resolve_name("b-service"), None, "C should see tombstone after B's death propagates");
}
// If SWIM hasn't declared death yet, the test still passes — the mechanism
// is wired, just needs more ticks. The important thing: no panics, clean flow.
}
// ─── Test 10: registry events emitted on change ─────────────────────────────
#[test]
fn registry_events_emitted_on_change() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10040"));
let actor = ActorAddress::new_random();
let node_id = node.node_id();
node.register_name("evt-test".into(), actor);
node.unregister_name("evt-test");
let events = node.registry_events();
assert_eq!(events.len(), 2);
assert_eq!(
events[0],
RegistryEvent::Registered {
name: "evt-test".into(),
actor_addr: actor,
node_id,
}
);
assert!(matches!(
&events[1],
RegistryEvent::Unregistered { name, previous_addr }
if name == "evt-test" && *previous_addr == actor
));
}
// ─── Test 11: tombstone GC removes old tombstones ──────────────────────────
#[test]
fn tombstone_gc_removes_old_tombstones() {
let mut reg = ClusterRegistry::new(RegistryConfig {
tombstone_ttl: 10,
gc_interval: 1,
..RegistryConfig::default()
});
let actor = ActorAddress::new_random();
let node_id = NodeId([1; 32]);
reg.register("gc-me".into(), actor, node_id, 1);
reg.unregister("gc-me", node_id, 1);
// Tombstone exists.
assert_eq!(reg.resolve("gc-me"), None);
assert_eq!(reg.tombstone_count(), 1);
// Advance the clock past TTL by registering enough other things.
// Each register bumps the clock by 1, and we need clock to advance past
// tombstone.timestamp + tombstone_ttl.
for i in 0..15 {
let a = ActorAddress::new_random();
reg.register(format!("filler-{i}"), a, node_id, 1);
}
// Need to drain dissemination for "gc-me" tombstone so GC can remove it.
for _ in 0..20 {
reg.take_pending(100);
}
// Now run GC.
reg.gc_tick();
// The tombstone should be gone.
assert_eq!(reg.tombstone_count(), 0, "tombstone should be GC'd after TTL");
}
// ─── Test 12: gossip convergence with five nodes ────────────────────────────
#[test]
fn gossip_convergence_five_nodes() {
let base_port = 10050;
let mut nodes: Vec<DistributedNode> = (0..5)
.map(|i| {
DistributedNode::new(test_config(&format!("127.0.0.1:{}", base_port + i)))
})
.collect();
// Collect ids/addrs before joining (borrow gymnastics).
let ids: Vec<NodeId> = nodes.iter().map(|n| n.node_id()).collect();
let addrs: Vec<SocketAddr> = nodes.iter().map(|n| n.listen_addr()).collect();
// All join through node 0.
for i in 1..5 {
let actions = nodes[i].join(&[addrs[0]]);
// Deliver join request to node 0.
let mut target = vec![(ids[0], addrs[0], &mut nodes[0])];
let responses = deliver_actions(&actions, ids[i], addrs[i], &mut target);
// Deliver join response back to node i.
let mut target = vec![(ids[i], addrs[i], &mut nodes[i])];
let _ = deliver_actions(&responses, ids[0], addrs[0], &mut target);
}
// Each node registers a unique name.
let actors: Vec<ActorAddress> = (0..5).map(|_| ActorAddress::new_random()).collect();
for i in 0..5 {
nodes[i].register_name(format!("service-{i}"), actors[i]);
}
// Run many gossip rounds between all pairs.
for _round in 0..15 {
for i in 0..5 {
let tick_actions = nodes[i].tick();
// Deliver to all other nodes.
for j in 0..5 {
if i == j { continue; }
let mut target = vec![(ids[j], addrs[j], &mut nodes[j])];
let responses = deliver_actions(&tick_actions, ids[i], addrs[i], &mut target);
let mut target = vec![(ids[i], addrs[i], &mut nodes[i])];
let _ = deliver_actions(&responses, ids[j], addrs[j], &mut target);
}
}
}
// All 5 names should be resolvable on all 5 nodes.
for i in 0..5 {
for j in 0..5 {
let result = nodes[i].resolve_name(&format!("service-{j}"));
assert_eq!(
result,
Some((actors[j], ids[j])),
"node {i} should resolve service-{j}"
);
}
}
}

View file

@ -14,6 +14,7 @@ crossbeam-queue = "0.3.12"
ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] } ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] }
crossterm = { version = "0.28", optional = true } crossterm = { version = "0.28", optional = true }
distribution = { path = "../distribution", optional = true } distribution = { path = "../distribution", optional = true }
swactor-command = { path = "../command" }
[dependencies.ctrlc] [dependencies.ctrlc]
version = "3" version = "3"

View file

@ -291,6 +291,7 @@ fn main() {
swim: swim_config.clone(), swim: swim_config.clone(),
cache_capacity: if i == 0 { 1000 } else { 100 }, cache_capacity: if i == 0 { 1000 } else { 100 },
republish_interval: 500, republish_interval: 500,
..Default::default()
}; };
let node = DistributedNode::new(config); let node = DistributedNode::new(config);
node_ids.push(node.node_id()); node_ids.push(node.node_id());
@ -448,6 +449,7 @@ fn main() {
swim: swim_config.clone(), swim: swim_config.clone(),
cache_capacity: 100, cache_capacity: 100,
republish_interval: 500, republish_interval: 500,
..Default::default()
}; };
let revived = DistributedNode::new(config); let revived = DistributedNode::new(config);
let join_actions = revived.join(&[seed_addr]); let join_actions = revived.join(&[seed_addr]);
@ -512,6 +514,7 @@ fn main() {
swim: swim_config.clone(), swim: swim_config.clone(),
cache_capacity: 100, cache_capacity: 100,
republish_interval: 500, republish_interval: 500,
..Default::default()
}; };
let revived = DistributedNode::new(config); let revived = DistributedNode::new(config);
let join_actions = revived.join(&[seed_addr]); let join_actions = revived.join(&[seed_addr]);

View file

@ -42,6 +42,12 @@ impl StatsCollector {
} }
} }
impl swactor_command::StatsEnricher for StatsCollector {
fn enrich(&self, stats: &mut swactor::stats::RuntimeStats) {
stats.actor_details = self.actor_details();
}
}
impl StatsHook for StatsCollector { impl StatsHook for StatsCollector {
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) { fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) {
if let Some(slot) = self.slots.get(worker_id) { if let Some(slot) = self.slots.get(worker_id) {

View file

@ -1,5 +1,7 @@
//! Line-oriented diagnostic protocol for LLM-driven runtime investigation. //! Line-oriented diagnostic protocol for LLM-driven runtime investigation.
//! //!
//! Delegates all command logic to the `swactor-command` crate.
//!
//! Send text commands on stdin, receive JSON responses on stdout (one per line). //! Send text commands on stdin, receive JSON responses on stdout (one per line).
//! All human-readable diagnostics go to stderr. //! All human-readable diagnostics go to stderr.
//! //!
@ -19,16 +21,17 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::io::{self, BufRead, Write}; use std::io::{self, BufRead, Write};
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant};
use serde::Serialize;
use swactor::runtime::Runtime; use swactor::runtime::Runtime;
use swactor::stats::RuntimeStats; use swactor_command::{CommandContext, CommandRouter};
use crate::collector::StatsCollector; use crate::collector::StatsCollector;
/// Run the investigate REPL. Blocks until stdin is closed or `quit` is received. /// Run the investigate REPL. Blocks until stdin is closed or `quit` is received.
pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) -> io::Result<()> { pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) -> io::Result<()> {
let router = CommandRouter::with_builtins();
let ctx = CommandContext::with_enricher(runtime, collector);
let stdin = io::stdin(); let stdin = io::stdin();
let mut stdout = io::stdout(); let mut stdout = io::stdout();
@ -40,18 +43,14 @@ pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) ->
if line.is_empty() { if line.is_empty() {
continue; continue;
} }
if line == "quit" || line == "exit" {
let parts: Vec<&str> = line.split_whitespace().collect();
let cmd = parts[0];
let args = &parts[1..];
if cmd == "quit" || cmd == "exit" {
break; break;
} }
let response = dispatch_repl(cmd, args, &runtime, &collector); let req = swactor_command::parse_line(line);
let resp = router.dispatch(&req, &ctx);
stdout.write_all(response.as_bytes())?; stdout.write_all(resp.to_json_line().as_bytes())?;
stdout.write_all(b"\n")?; stdout.write_all(b"\n")?;
stdout.flush()?; stdout.flush()?;
} }
@ -59,501 +58,14 @@ pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) ->
Ok(()) Ok(())
} }
fn dispatch_repl(cmd: &str, args: &[&str], runtime: &Runtime, collector: &StatsCollector) -> String {
match cmd {
"help" => cmd_help(),
"overview" => cmd_overview(runtime, collector),
"workers" => cmd_workers(runtime),
"worker" => cmd_worker(runtime, collector, args),
"actors" => cmd_actors(runtime, collector, args),
"actor" => cmd_actor(runtime, collector, args),
"hot" => cmd_hot(runtime, collector, args),
"phases" => cmd_phases(runtime, args),
"diff" => cmd_diff(runtime, collector, args),
_ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")),
}
}
/// Dispatch an investigate command from HTTP query parameters. /// Dispatch an investigate command from HTTP query parameters.
/// ///
/// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command function. /// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command.
pub fn dispatch_command( pub fn dispatch_command(
cmd: &str,
params: &HashMap<String, String>, params: &HashMap<String, String>,
runtime: &Runtime, router: &CommandRouter,
collector: &StatsCollector, ctx: &CommandContext,
) -> String { ) -> String {
match cmd { let req = swactor_command::from_query_params(params);
"help" => cmd_help(), router.dispatch(&req, ctx).to_json_line()
"overview" => cmd_overview(runtime, collector),
"workers" => cmd_workers(runtime),
"worker" => {
let id = params.get("id").map(|s| s.as_str()).unwrap_or("");
cmd_worker(runtime, collector, &[id])
}
"actors" => {
let mut args = Vec::new();
if let Some(sort) = params.get("sort") {
args.push("--sort");
args.push(sort.as_str());
}
if let Some(limit) = params.get("limit") {
args.push("--limit");
args.push(limit.as_str());
}
if let Some(worker) = params.get("worker") {
args.push("--worker");
args.push(worker.as_str());
}
cmd_actors(runtime, collector, &args)
}
"actor" => {
let prefix = params.get("prefix").map(|s| s.as_str()).unwrap_or("");
cmd_actor(runtime, collector, &[prefix])
}
"hot" => {
let n = params.get("n").map(|s| s.as_str()).unwrap_or("10");
cmd_hot(runtime, collector, &[n])
}
"phases" => {
match params.get("worker") {
Some(w) => cmd_phases(runtime, &[w.as_str()]),
None => cmd_phases(runtime, &[]),
}
}
"diff" => {
let secs = params.get("seconds").map(|s| s.as_str()).unwrap_or("");
cmd_diff(runtime, collector, &[secs])
}
_ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")),
}
}
// ── Helpers ─────────────────────────────────────────────────────────────
fn ok_response(cmd: &str, data: impl Serialize) -> String {
serde_json::to_string(&serde_json::json!({
"ok": true,
"command": cmd,
"data": data,
}))
.unwrap_or_else(|e| err_response(cmd, &format!("serialization error: {e}")))
}
fn err_response(cmd: &str, msg: &str) -> String {
serde_json::to_string(&serde_json::json!({
"ok": false,
"command": cmd,
"error": msg,
}))
.unwrap()
}
fn format_addr(addr: &swactor::actor::ActorAddress) -> String {
format!("{addr}")
}
fn full_hex(addr: &swactor::actor::ActorAddress) -> String {
addr.0.iter().map(|b| format!("{b:02x}")).collect()
}
fn enriched_stats(rt: &Runtime, col: &StatsCollector) -> RuntimeStats {
let mut s = rt.stats();
col.enrich(&mut s);
s
}
// ── Commands ────────────────────────────────────────────────────────────
pub fn cmd_help() -> String {
ok_response(
"help",
serde_json::json!({
"commands": [
{"name": "overview", "usage": "overview", "description": "Summary: worker count, actor count, total messages, mailbox depth, panics"},
{"name": "workers", "usage": "workers", "description": "Per-worker stats: actors, mailbox depth, messages, sends (local/cross/inbox), panics"},
{"name": "worker", "usage": "worker <id>", "description": "Single worker detail with tick-phase timing breakdown"},
{"name": "actors", "usage": "actors [--sort mailbox|worker|address] [--limit N] [--worker W]", "description": "List actors with optional sorting, limit, and worker filter"},
{"name": "actor", "usage": "actor <hex_prefix>", "description": "Find actor(s) whose address starts with the given hex prefix"},
{"name": "hot", "usage": "hot [N]", "description": "Top N actors by mailbox depth (default 10)"},
{"name": "phases", "usage": "phases [worker_id]", "description": "Tick-phase time breakdown (all workers or one)"},
{"name": "diff", "usage": "diff <seconds>", "description": "Collect two snapshots N seconds apart, report deltas and rates"},
{"name": "quit", "usage": "quit", "description": "Exit the investigate session"},
]
}),
)
}
pub fn cmd_overview(rt: &Runtime, col: &StatsCollector) -> String {
let stats = enriched_stats(rt, col);
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum();
let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum();
let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum();
let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum();
let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum();
let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum();
ok_response(
"overview",
serde_json::json!({
"workers": stats.num_workers,
"actors": stats.actor_details.len(),
"total_messages_processed": total_msgs,
"total_mailbox_depth": total_mailbox,
"total_panics": total_panics,
"total_type_mismatches": total_type_mismatches,
"sends": {
"local": total_local,
"cross_worker": total_cross,
"inbox": total_inbox,
},
}),
)
}
pub fn cmd_workers(rt: &Runtime) -> String {
let stats = rt.stats();
let workers: Vec<_> = stats
.workers
.iter()
.map(|w| {
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
})
})
.collect();
ok_response("workers", workers)
}
pub fn cmd_worker(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let id: usize = match args.first().and_then(|s| s.parse().ok()) {
Some(id) => id,
None => return err_response("worker", "usage: worker <id>"),
};
let stats = enriched_stats(rt, col);
let w = match stats.workers.iter().find(|w| w.id == id) {
Some(w) => w,
None => {
return err_response(
"worker",
&format!("worker {id} not found (have 0..{})", stats.num_workers),
)
}
};
// Tick phase breakdown for this worker
let timings = stats.tick_timings.get(id).cloned().unwrap_or_default();
let phase_breakdown = compute_phase_breakdown(&timings);
let actors_on_worker: Vec<_> = stats
.actor_details
.iter()
.filter(|a| a.worker_id == id)
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response(
"worker",
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
"tick_phases": phase_breakdown,
"actor_details": actors_on_worker,
}),
)
}
pub fn cmd_actors(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let stats = enriched_stats(rt, col);
let mut actors = stats.actor_details.clone();
// Parse flags
let mut sort_by = "mailbox";
let mut limit: usize = usize::MAX;
let mut worker_filter: Option<usize> = None;
let mut i = 0;
while i < args.len() {
match args[i] {
"--sort" if i + 1 < args.len() => {
sort_by = args[i + 1];
i += 2;
}
"--limit" if i + 1 < args.len() => {
limit = args[i + 1].parse().unwrap_or(usize::MAX);
i += 2;
}
"--worker" if i + 1 < args.len() => {
worker_filter = args[i + 1].parse().ok();
i += 2;
}
_ => {
i += 1;
}
}
}
if let Some(wid) = worker_filter {
actors.retain(|a| a.worker_id == wid);
}
match sort_by {
"mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)),
"worker" => actors.sort_by_key(|a| a.worker_id),
"address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)),
other => return err_response("actors", &format!("unknown sort field `{other}` — use mailbox|worker|address")),
}
actors.truncate(limit);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response(
"actors",
serde_json::json!({
"total": stats.actor_details.len(),
"returned": rows.len(),
"sort": sort_by,
"actors": rows,
}),
)
}
pub fn cmd_actor(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let prefix = match args.first() {
Some(p) => *p,
None => return err_response("actor", "usage: actor <hex_prefix>"),
};
let stats = enriched_stats(rt, col);
let matches: Vec<_> = stats
.actor_details
.iter()
.filter(|a| full_hex(&a.address).starts_with(prefix))
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response(
"actor",
serde_json::json!({
"prefix": prefix,
"matches": matches.len(),
"actors": matches,
}),
)
}
pub fn cmd_hot(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let n: usize = args.first().and_then(|s| s.parse().ok()).unwrap_or(10);
let stats = enriched_stats(rt, col);
let mut actors = stats.actor_details.clone();
actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth));
actors.truncate(n);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response("hot", rows)
}
pub fn cmd_phases(rt: &Runtime, args: &[&str]) -> String {
let stats = rt.stats();
let worker_filter: Option<usize> = args.first().and_then(|s| s.parse().ok());
let phase_names = [
"spawn_drain",
"transfer_drain",
"tick_all",
"spawn_drain_2",
"pending_local",
"stats_publish",
];
let mut results = Vec::new();
for (i, timings) in stats.tick_timings.iter().enumerate() {
if let Some(wid) = worker_filter {
if i != wid {
continue;
}
}
let breakdown = compute_phase_breakdown(timings);
results.push(serde_json::json!({
"worker_id": i,
"ticks_sampled": timings.len(),
"phases": breakdown,
"phase_names": phase_names,
}));
}
ok_response("phases", results)
}
pub fn cmd_diff(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let secs: f64 = match args.first().and_then(|s| s.parse().ok()) {
Some(s) if s > 0.0 && s <= 30.0 => s,
Some(_) => return err_response("diff", "seconds must be between 0 and 30"),
None => return err_response("diff", "usage: diff <seconds>"),
};
let before = enriched_stats(rt, col);
let t0 = Instant::now();
std::thread::sleep(Duration::from_secs_f64(secs));
let after = enriched_stats(rt, col);
let elapsed = t0.elapsed().as_secs_f64();
let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum();
let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum();
let delta_msgs = msgs_after.saturating_sub(msgs_before);
let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum();
let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum();
let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum();
let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum();
let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum();
let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum();
let per_worker: Vec<_> = after
.workers
.iter()
.enumerate()
.map(|(i, w)| {
let prev = before.workers.get(i);
let d = prev
.map(|p| w.messages_processed.saturating_sub(p.messages_processed))
.unwrap_or(0);
serde_json::json!({
"worker_id": i,
"delta_messages": d,
"msg_per_sec": d as f64 / elapsed,
"actors_before": prev.map(|p| p.num_actors).unwrap_or(0),
"actors_after": w.num_actors,
"mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0),
"mailbox_after": w.mailbox_depth,
})
})
.collect();
ok_response(
"diff",
serde_json::json!({
"elapsed_s": elapsed,
"actors_before": before.actor_details.len(),
"actors_after": after.actor_details.len(),
"delta_messages": delta_msgs,
"msg_per_sec": delta_msgs as f64 / elapsed,
"delta_local_sends": local_after.saturating_sub(local_before),
"delta_cross_sends": cross_after.saturating_sub(cross_before),
"mailbox_before": mailbox_before,
"mailbox_after": mailbox_after,
"per_worker": per_worker,
}),
)
}
// ── Phase breakdown helper ──────────────────────────────────────────────
fn compute_phase_breakdown(
timings: &[swactor::stats::TickTiming],
) -> serde_json::Value {
if timings.is_empty() {
return serde_json::json!({
"ticks": 0,
"active_pct": 0.0,
"avg_tick_us": 0.0,
"phases_us": [0, 0, 0, 0, 0, 0],
"phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
});
}
let n = timings.len();
let active = timings.iter().filter(|t| t.did_work).count();
let active_pct = (active as f64 / n as f64) * 100.0;
let mut phase_sums = [0u64; 6];
for t in timings {
for (i, &us) in t.phase_us.iter().enumerate() {
phase_sums[i] += us;
}
}
let total_us: u64 = phase_sums.iter().sum();
let avg_tick_us = total_us as f64 / n as f64;
let phases_pct: Vec<f64> = if total_us == 0 {
vec![0.0; 6]
} else {
phase_sums
.iter()
.map(|&s| (s as f64 / total_us as f64) * 100.0)
.collect()
};
serde_json::json!({
"ticks": n,
"active_pct": active_pct,
"avg_tick_us": avg_tick_us,
"phases_us": phase_sums,
"phases_pct": phases_pct,
})
} }

View file

@ -11,7 +11,6 @@ use swactor::runtime::Runtime;
use crate::actors_html::ACTORS_HTML; use crate::actors_html::ACTORS_HTML;
use crate::collector::StatsCollector; use crate::collector::StatsCollector;
use crate::dashboard_html::DASHBOARD_HTML; use crate::dashboard_html::DASHBOARD_HTML;
use crate::investigate;
use crate::layer::EventStore; use crate::layer::EventStore;
use crate::trace::RuntimeTrace; use crate::trace::RuntimeTrace;
@ -127,6 +126,7 @@ pub(crate) fn spawn_http_server(
let addr = format!("0.0.0.0:{port}"); let addr = format!("0.0.0.0:{port}");
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server"); let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
let server = Arc::new(server); let server = Arc::new(server);
let cmd_router = Arc::new(swactor_command::CommandRouter::with_builtins());
for _ in 0..4 { for _ in 0..4 {
let server = Arc::clone(&server); let server = Arc::clone(&server);
@ -134,6 +134,7 @@ pub(crate) fn spawn_http_server(
let runtime = Arc::clone(&runtime); let runtime = Arc::clone(&runtime);
let collector = Arc::clone(&collector); let collector = Arc::clone(&collector);
let shutdown = Arc::clone(&shutdown); let shutdown = Arc::clone(&shutdown);
let cmd_router = Arc::clone(&cmd_router);
#[cfg(feature = "distribution")] #[cfg(feature = "distribution")]
let distribution = Arc::clone(&distribution); let distribution = Arc::clone(&distribution);
thread::spawn(move || { thread::spawn(move || {
@ -174,6 +175,7 @@ pub(crate) fn spawn_http_server(
&url, &url,
Arc::clone(&runtime), Arc::clone(&runtime),
Arc::clone(&collector), Arc::clone(&collector),
Arc::clone(&cmd_router),
); );
} }
_ => respond_404(request), _ => respond_404(request),
@ -283,21 +285,33 @@ fn handle_investigate_api(
url: &str, url: &str,
runtime: Arc<Mutex<Option<Arc<Runtime>>>>, runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>, collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
cmd_router: Arc<swactor_command::CommandRouter>,
) { ) {
let params = parse_query_string(url); let params = parse_query_string(url);
let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help");
let maybe_rt = runtime.lock().unwrap().clone(); let maybe_rt = runtime.lock().unwrap().clone();
let maybe_col = collector.lock().unwrap().clone(); let maybe_col = collector.lock().unwrap().clone();
let json = match (maybe_rt, maybe_col) { let json = match (maybe_rt, maybe_col) {
(Some(rt), Some(col)) => investigate::dispatch_command(cmd, &params, &rt, &col), (Some(rt), Some(col)) => {
_ => serde_json::json!({ let ctx = swactor_command::CommandContext::with_enricher(rt, col);
"ok": false, let req = swactor_command::from_query_params(&params);
"command": cmd, cmd_router.dispatch(&req, &ctx).to_json_line()
"error": "runtime not attached yet" }
}) (Some(rt), None) => {
.to_string(), let ctx = swactor_command::CommandContext::new(rt);
let req = swactor_command::from_query_params(&params);
cmd_router.dispatch(&req, &ctx).to_json_line()
}
_ => {
let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help");
serde_json::json!({
"ok": false,
"command": cmd,
"error": "runtime not attached yet"
})
.to_string()
}
}; };
let response = tiny_http::Response::from_string(json).with_header( let response = tiny_http::Response::from_string(json).with_header(

View file

@ -73,6 +73,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
swim: config.swim.clone(), swim: config.swim.clone(),
cache_capacity: config.cache_capacity, cache_capacity: config.cache_capacity,
republish_interval: 50, republish_interval: 50,
..Default::default()
}; };
let node = DistributedNode::new(node_config); let node = DistributedNode::new(node_config);
node_ids.push(node.node_id()); node_ids.push(node.node_id());
@ -184,6 +185,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
swim: config.swim.clone(), swim: config.swim.clone(),
cache_capacity: config.cache_capacity, cache_capacity: config.cache_capacity,
republish_interval: 50, republish_interval: 50,
..Default::default()
}; };
let revived = DistributedNode::new(node_config); let revived = DistributedNode::new(node_config);
// Rejoin the cluster. // Rejoin the cluster.

View file

@ -0,0 +1,12 @@
[package]
name = "swactor-wasm-actor"
version = "0.1.0"
edition = "2024"
[dependencies]
swactor = { path = "../.." }
wasmtime = "29"
[dev-dependencies]
swactor = { path = "../..", features = ["getrandom"] }
wat = "1"

View file

@ -0,0 +1,59 @@
use swactor::actor::{ActorInterface, Ctx};
use wasmtime::{Memory, Store, TypedFunc};
use crate::ByteMessage;
/// State accessible to host functions during guest execution.
#[derive(Default)]
pub(crate) struct HostState {
pub outbox: Vec<(swactor::actor::ActorAddress, Vec<u8>)>,
}
/// An actor whose logic is defined by a WebAssembly guest module.
///
/// Messages arrive as [`ByteMessage`], are copied into Wasm linear memory,
/// and processed by the guest's `handle` export. The guest can send messages
/// back via the `swactor.send` host import.
pub struct WasmActor {
pub(crate) store: Store<HostState>,
pub(crate) memory: Memory,
pub(crate) alloc: TypedFunc<i32, i32>,
pub(crate) handle: TypedFunc<(i32, i32), ()>,
}
impl ActorInterface for WasmActor {
type Incoming = ByteMessage;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: ByteMessage) {
let bytes = &msg.0;
let len: i32 = match i32::try_from(bytes.len()) {
Ok(n) => n,
Err(_) => return, // message too large for i32 ABI
};
// 1. Allocate space in guest memory
let ptr = match self.alloc.call(&mut self.store, len) {
Ok(ptr) if ptr < 0 => return, // invalid pointer
Ok(0) if len > 0 => return, // OOM — drop message
Ok(ptr) => ptr,
Err(_) => return, // alloc trapped — drop message
};
// 2. Write message bytes into guest memory
self.memory.data_mut(&mut self.store)
[ptr as usize..(ptr as usize + bytes.len())]
.copy_from_slice(bytes);
// 3. Call guest handle
if self.handle.call(&mut self.store, (ptr, len)).is_err() {
return; // handle trapped — drop message, keep actor alive
}
// 4. Drain outbox → send via ctx
let outbox: Vec<_> = self.store.data_mut().outbox.drain(..).collect();
for (dest, payload) in outbox {
let _ = ctx.send(dest, ByteMessage(payload));
}
}
}

View file

@ -0,0 +1,109 @@
use swactor::actor::ActorAddress;
use wasmtime::{Linker, Module, Store, TypedFunc};
use crate::actor::{HostState, WasmActor};
use crate::engine::SharedEngine;
use crate::error::WasmActorError;
/// Compiles a Wasm module and produces a ready-to-use [`WasmActor`].
pub struct WasmActorBuilder {
engine: SharedEngine,
wasm_bytes: Vec<u8>,
}
impl WasmActorBuilder {
pub fn new(engine: SharedEngine, wasm_bytes: impl Into<Vec<u8>>) -> Self {
Self {
engine,
wasm_bytes: wasm_bytes.into(),
}
}
/// Compile the module, link host functions, and instantiate.
pub fn build(self) -> Result<WasmActor, WasmActorError> {
let engine = self.engine.inner();
let module = Module::new(engine, &self.wasm_bytes)?;
let mut linker: Linker<HostState> = Linker::new(engine);
Self::link_send(&mut linker)?;
let mut store = Store::new(engine, HostState::default());
let instance = linker.instantiate(&mut store, &module)?;
// Extract required exports
let memory = instance
.get_memory(&mut store, "memory")
.ok_or(WasmActorError::MissingExport("memory"))?;
let alloc: TypedFunc<i32, i32> = instance
.get_typed_func(&mut store, "alloc")
.map_err(|_| WasmActorError::MissingExport("alloc"))?;
let handle: TypedFunc<(i32, i32), ()> = instance
.get_typed_func(&mut store, "handle")
.map_err(|_| WasmActorError::MissingExport("handle"))?;
Ok(WasmActor {
store,
memory,
alloc,
handle,
})
}
/// Link the `swactor.send` host import.
fn link_send(linker: &mut Linker<HostState>) -> Result<(), WasmActorError> {
linker.func_wrap(
"swactor",
"send",
|mut caller: wasmtime::Caller<'_, HostState>,
dest_ptr: i32,
payload_ptr: i32,
payload_len: i32|
-> Result<(), wasmtime::Error> {
let mem = caller
.get_export("memory")
.and_then(|e| e.into_memory())
.ok_or_else(|| wasmtime::Error::msg("guest must export memory"))?;
let data = mem.data(&caller);
let mem_len = data.len();
// Validate non-negative arguments
if dest_ptr < 0 || payload_ptr < 0 || payload_len < 0 {
return Err(wasmtime::Error::msg(
"negative argument in swactor.send",
));
}
let dest_ptr = dest_ptr as usize;
let payload_ptr = payload_ptr as usize;
let payload_len = payload_len as usize;
// Bounds-check with overflow protection
let dest_end = dest_ptr
.checked_add(32)
.ok_or_else(|| wasmtime::Error::msg("dest_ptr overflow"))?;
let payload_end = payload_ptr
.checked_add(payload_len)
.ok_or_else(|| wasmtime::Error::msg("payload range overflow"))?;
if dest_end > mem_len || payload_end > mem_len {
return Err(wasmtime::Error::msg(
"out-of-bounds memory access in swactor.send",
));
}
// Read 32-byte destination address
let mut addr_bytes = [0u8; 32];
addr_bytes.copy_from_slice(&data[dest_ptr..dest_end]);
let dest = ActorAddress(addr_bytes);
// Read payload
let payload = data[payload_ptr..payload_end].to_vec();
caller.data_mut().outbox.push((dest, payload));
Ok(())
},
)?;
Ok(())
}
}

View file

@ -0,0 +1,35 @@
use std::sync::Arc;
use wasmtime::Engine;
/// A shared, cheaply-cloneable Wasm engine.
///
/// Created once and reused across multiple [`WasmActor`](crate::WasmActor) instances.
/// Configured with maximum sandboxing — no threads, no SIMD, no reference types.
#[derive(Clone)]
pub struct SharedEngine(Arc<Engine>);
impl std::fmt::Debug for SharedEngine {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("SharedEngine").field(&"<Engine>").finish()
}
}
impl SharedEngine {
/// Create a new engine with sandboxed defaults.
pub fn new() -> Result<Self, wasmtime::Error> {
let mut config = wasmtime::Config::new();
config.wasm_threads(false);
config.wasm_simd(false);
config.wasm_relaxed_simd(false);
config.wasm_reference_types(false);
config.wasm_multi_value(false);
config.wasm_bulk_memory(true);
let engine = Engine::new(&config)?;
Ok(Self(Arc::new(engine)))
}
pub(crate) fn inner(&self) -> &Engine {
&self.0
}
}

View file

@ -0,0 +1,27 @@
use std::fmt;
/// Errors that can occur when building or running a WasmActor.
#[derive(Debug)]
pub enum WasmActorError {
/// A required export is missing from the Wasm module.
MissingExport(&'static str),
/// The Wasm module failed to compile or instantiate.
Wasmtime(wasmtime::Error),
}
impl fmt::Display for WasmActorError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::MissingExport(name) => write!(f, "missing required export: `{name}`"),
Self::Wasmtime(e) => write!(f, "wasmtime error: {e}"),
}
}
}
impl std::error::Error for WasmActorError {}
impl From<wasmtime::Error> for WasmActorError {
fn from(e: wasmtime::Error) -> Self {
Self::Wasmtime(e)
}
}

View file

@ -0,0 +1,13 @@
mod actor;
mod builder;
mod engine;
mod error;
pub use actor::WasmActor;
pub use builder::WasmActorBuilder;
pub use engine::SharedEngine;
pub use error::WasmActorError;
/// A message carrying raw bytes, suitable for passing to/from Wasm guests.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ByteMessage(pub Vec<u8>);

View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "double-guest"
version = "0.1.0"

View file

@ -0,0 +1,13 @@
[workspace]
[package]
name = "double-guest"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["cdylib"]
[profile.release]
opt-level = "s"
lto = true

View file

@ -0,0 +1,63 @@
#![no_std]
use core::cell::UnsafeCell;
use core::panic::PanicInfo;
// --- bump allocator ---
const HEAP_SIZE: usize = 65536;
struct BumpAlloc {
heap: UnsafeCell<[u8; HEAP_SIZE]>,
offset: UnsafeCell<usize>,
}
unsafe impl Sync for BumpAlloc {}
static ALLOC: BumpAlloc = BumpAlloc {
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
offset: UnsafeCell::new(0),
};
#[unsafe(no_mangle)]
pub extern "C" fn alloc(size: i32) -> i32 {
unsafe {
let offset = &mut *ALLOC.offset.get();
let heap = &mut *ALLOC.heap.get();
let align = 8;
let start = (*offset + align - 1) & !(align - 1);
let end = start + size as usize;
if end > heap.len() {
return 0; // OOM
}
*offset = end;
heap.as_ptr().add(start) as i32
}
}
// --- host import ---
#[link(wasm_import_module = "swactor")]
unsafe extern "C" {
#[link_name = "send"]
fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32);
}
/// Message format: first 32 bytes = destination address, rest = payload.
/// Sends the payload back twice to demonstrate multi-send.
#[unsafe(no_mangle)]
pub extern "C" fn handle(ptr: i32, len: i32) {
if len < 32 {
return;
}
let dest_ptr = ptr;
let payload_ptr = ptr + 32;
let payload_len = len - 32;
unsafe {
host_send(dest_ptr, payload_ptr, payload_len);
host_send(dest_ptr, payload_ptr, payload_len);
}
}
#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
loop {}
}

View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "echo-guest"
version = "0.1.0"

View file

@ -0,0 +1,13 @@
[workspace]
[package]
name = "echo-guest"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["cdylib"]
[profile.release]
opt-level = "s"
lto = true

View file

@ -0,0 +1,62 @@
#![no_std]
use core::cell::UnsafeCell;
use core::panic::PanicInfo;
// --- bump allocator ---
const HEAP_SIZE: usize = 65536;
struct BumpAlloc {
heap: UnsafeCell<[u8; HEAP_SIZE]>,
offset: UnsafeCell<usize>,
}
unsafe impl Sync for BumpAlloc {}
static ALLOC: BumpAlloc = BumpAlloc {
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
offset: UnsafeCell::new(0),
};
#[unsafe(no_mangle)]
pub extern "C" fn alloc(size: i32) -> i32 {
unsafe {
let offset = &mut *ALLOC.offset.get();
let heap = &mut *ALLOC.heap.get();
let align = 8;
let start = (*offset + align - 1) & !(align - 1);
let end = start + size as usize;
if end > heap.len() {
return 0; // OOM
}
*offset = end;
heap.as_ptr().add(start) as i32
}
}
// --- host import ---
#[link(wasm_import_module = "swactor")]
unsafe extern "C" {
#[link_name = "send"]
fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32);
}
/// Message format: first 32 bytes = destination address, rest = payload.
/// Echo sends the payload portion back to the specified destination.
#[unsafe(no_mangle)]
pub extern "C" fn handle(ptr: i32, len: i32) {
if len < 32 {
return;
}
let dest_ptr = ptr;
let payload_ptr = ptr + 32;
let payload_len = len - 32;
unsafe {
host_send(dest_ptr, payload_ptr, payload_len);
}
}
#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
loop {}
}

View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "silent-guest"
version = "0.1.0"

View file

@ -0,0 +1,13 @@
[workspace]
[package]
name = "silent-guest"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["cdylib"]
[profile.release]
opt-level = "s"
lto = true

View file

@ -0,0 +1,45 @@
#![no_std]
use core::cell::UnsafeCell;
use core::panic::PanicInfo;
// --- bump allocator ---
const HEAP_SIZE: usize = 65536;
struct BumpAlloc {
heap: UnsafeCell<[u8; HEAP_SIZE]>,
offset: UnsafeCell<usize>,
}
unsafe impl Sync for BumpAlloc {}
static ALLOC: BumpAlloc = BumpAlloc {
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
offset: UnsafeCell::new(0),
};
#[unsafe(no_mangle)]
pub extern "C" fn alloc(size: i32) -> i32 {
unsafe {
let offset = &mut *ALLOC.offset.get();
let heap = &mut *ALLOC.heap.get();
let align = 8;
let start = (*offset + align - 1) & !(align - 1);
let end = start + size as usize;
if end > heap.len() {
return 0; // OOM
}
*offset = end;
heap.as_ptr().add(start) as i32
}
}
#[unsafe(no_mangle)]
pub extern "C" fn handle(_ptr: i32, _len: i32) {
// Silent: receive bytes, do nothing
}
#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
loop {}
}

View file

@ -0,0 +1,331 @@
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder, WasmActorError};
fn guest_wasm(name: &str) -> Vec<u8> {
let path = format!(
"{}/tests/guests/{name}/target/wasm32-unknown-unknown/release/{name}_guest.wasm",
env!("CARGO_MANIFEST_DIR")
);
std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {path}: {e}"))
}
/// Build a message with an inbox address prepended (the guest contract).
fn framed_msg(dest: &ActorAddress, payload: &[u8]) -> ByteMessage {
let mut buf = Vec::with_capacity(32 + payload.len());
buf.extend_from_slice(&dest.0);
buf.extend_from_slice(payload);
ByteMessage(buf)
}
// ── Echo: send bytes in, same bytes come back ────────────────────────────────
#[test]
fn echo_returns_same_payload() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("echo"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
let payload = b"hello wasm";
rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap();
rt.tick();
let received = inbox.try_recv().expect("inbox should have a message");
assert_eq!(received.0, payload);
}
#[test]
fn echo_preserves_binary_payload() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("echo"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
let payload: Vec<u8> = (0..=255).collect();
rt.send_to(addr, framed_msg(inbox.addr(), &payload)).unwrap();
rt.tick();
let received = inbox.try_recv().expect("inbox should have a message");
assert_eq!(received.0, payload);
}
// ── Silent: processes messages without sending anything ───────────────────────
#[test]
fn silent_produces_no_output() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("silent"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(b"ignored".to_vec())).unwrap();
rt.tick();
assert!(inbox.try_recv().is_none(), "silent guest should not send anything");
}
// ── Double: one message in, two messages out ─────────────────────────────────
#[test]
fn double_sends_two_copies() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("double"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
let payload = b"dup me";
rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap();
rt.tick();
let first = inbox.try_recv().expect("should receive first copy");
let second = inbox.try_recv().expect("should receive second copy");
assert_eq!(first.0, payload);
assert_eq!(second.0, payload);
assert!(inbox.try_recv().is_none(), "exactly two messages expected");
}
// ── Missing export → WasmActorError::MissingExport ───────────────────────────
#[test]
fn missing_alloc_export_returns_error() {
// Minimal valid Wasm module: (module) — no exports at all
let minimal_wasm = wat::parse_str("(module)").unwrap();
let engine = SharedEngine::new().unwrap();
let result = WasmActorBuilder::new(engine, minimal_wasm).build();
match result {
Err(WasmActorError::MissingExport(name)) => {
assert!(
name == "memory" || name == "alloc",
"expected missing memory or alloc, got: {name}"
);
}
Err(other) => panic!("expected MissingExport, got: {other}"),
Ok(_) => panic!("expected error for module with no exports"),
}
}
// ── Engine sharing: two actors from the same engine ──────────────────────────
#[test]
fn shared_engine_serves_multiple_actors() {
let engine = SharedEngine::new().unwrap();
let echo = WasmActorBuilder::new(engine.clone(), guest_wasm("echo"))
.build()
.unwrap();
let silent = WasmActorBuilder::new(engine, guest_wasm("silent"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let echo_addr = rt.spawn(echo).unwrap();
let _silent_addr = rt.spawn(silent).unwrap();
let payload = b"shared engine test";
rt.send_to(echo_addr, framed_msg(inbox.addr(), payload)).unwrap();
rt.tick();
let received = inbox.try_recv().expect("echo actor should still work");
assert_eq!(received.0, payload);
}
// ── Safety: edge cases that previously caused panics or corruption ────────────
#[test]
fn oob_send_traps_cleanly_and_actor_survives() {
// Guest calls swactor.send with dest_ptr pointing past the end of memory.
// The host should trap the call; the actor should survive for future messages.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const 0 ;; return start of memory (simplistic)
)
(func (export "handle") (param i32 i32)
;; Call send with dest_ptr = 65536 (1 page = end of memory, OOB for 32 bytes)
i32.const 65536
i32.const 0
i32.const 0
call $send
)
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
// Send a message — handle will try OOB send, which traps
rt.send_to(addr, ByteMessage(vec![42])).unwrap();
rt.tick();
// No message should arrive (the send was invalid)
assert!(inbox.try_recv().is_none(), "OOB send should not produce a message");
}
#[test]
fn alloc_oom_drops_message_actor_stays_alive() {
// Guest alloc always returns 0 (OOM). Message should be dropped,
// actor should remain alive for subsequent messages.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const 0 ;; always OOM
)
(func (export "handle") (param i32 i32)
;; Should never be called if alloc returned 0 for non-zero len
)
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
// Send a non-empty message — alloc returns 0, message should be dropped
rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap();
rt.tick();
// Actor is still alive — send another message, tick again (no panic)
rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap();
rt.tick();
}
#[test]
fn negative_alloc_ptr_drops_message() {
// Guest alloc returns -1. Host should detect the negative pointer and drop.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const -1 ;; invalid negative pointer
)
(func (export "handle") (param i32 i32))
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(vec![1])).unwrap();
rt.tick(); // should not panic
// Actor survives
rt.send_to(addr, ByteMessage(vec![2])).unwrap();
rt.tick();
}
#[test]
fn handle_trap_drops_message_actor_survives() {
// Guest handle executes `unreachable`, causing a Wasm trap.
// Message should be dropped, actor should stay alive.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const 256 ;; valid allocation
)
(func (export "handle") (param i32 i32)
unreachable ;; trap!
)
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap();
rt.tick(); // handle traps, but actor should survive
// Actor is still alive
rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap();
rt.tick();
}
// ── Integration: WasmActor alongside a native Rust actor ─────────────────────
#[derive(Clone)]
struct ForwardToWasm {
wasm_addr: ActorAddress,
inbox_addr: ActorAddress,
}
struct Forwarder;
impl ActorInterface for Forwarder {
type Incoming = ForwardToWasm;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: ForwardToWasm) {
// Build the framed message and forward to the wasm actor
let payload = b"from native";
let framed = framed_msg(&msg.inbox_addr, payload);
let _ = ctx.send(msg.wasm_addr, framed);
}
}
#[test]
fn native_actor_communicates_with_wasm_actor() {
let engine = SharedEngine::new().unwrap();
let wasm = WasmActorBuilder::new(engine, guest_wasm("echo"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let wasm_addr = rt.spawn(wasm).unwrap();
let forwarder_addr = rt.spawn(Forwarder).unwrap();
rt.send_to(
forwarder_addr,
ForwardToWasm {
wasm_addr,
inbox_addr: *inbox.addr(),
},
)
.unwrap();
// Tick 1: Forwarder receives message and sends to WasmActor
rt.tick();
// Tick 2: WasmActor receives the forwarded message and echoes to inbox
rt.tick();
let received = inbox.try_recv().expect("wasm actor should have echoed");
assert_eq!(received.0, b"from native");
}

View file

@ -0,0 +1,191 @@
# Wasm Actor Crate — Development History
> Adds a new crate (`crates/wasm-actor/`) that runs WebAssembly guest code
> **inside** a swactor actor. The Wasm instance lives in the actor — not as a
> separate OS process. Messages arrive as bytes, get written into Wasm linear
> memory, and the guest's `handle` export is called.
>
> ~350 lines of Rust (host) · 3 guest modules · 7 tests
---
## Table of Contents
1. [Overview & Motivation](#1-overview--motivation)
2. [What Was Built](#2-what-was-built)
3. [Guest ↔ Host Contract](#3-guest--host-contract)
4. [Handle Cycle (Hot Path)](#4-handle-cycle-hot-path)
5. [Guest Modules](#5-guest-modules)
6. [Design Decisions & Tradeoffs](#6-design-decisions--tradeoffs)
7. [Known Gaps & Future Improvements](#7-known-gaps--future-improvements)
8. [Test Coverage Summary](#8-test-coverage-summary)
---
## 1. Overview & Motivation
Swactor already supported running *inside* a browser via `crates/wasm/`
(wasm-bindgen). This crate flips the direction: run untrusted Wasm code
*inside* an actor, sandboxed by wasmtime. Use cases include user-defined
plugins, multi-language actors, and capability-restricted compute.
The main swactor crate has no wasmtime dependency — all Wasm machinery is
isolated in `crates/wasm-actor/`.
---
## 2. What Was Built
| Component | Location | Purpose |
|-----------|----------|---------|
| `swactor-wasm-actor` crate | `crates/wasm-actor/` | Host-side: engine, builder, actor impl |
| 3 guest crates | `crates/wasm-actor/tests/guests/{echo,double,silent}/` | `#![no_std]` Wasm modules for testing |
| Integration tests | `crates/wasm-actor/tests/wasm_actor.rs` | 7 behavioral tests |
### Crate modules
```
crates/wasm-actor/src/
lib.rs — ByteMessage, re-exports
engine.rs — SharedEngine (Arc<wasmtime::Engine>)
builder.rs — WasmActorBuilder (compile + link + instantiate)
actor.rs — WasmActor implementing ActorInterface
error.rs — WasmActorError enum
```
### Public types
- **`ByteMessage(pub Vec<u8>)`** — message type for Wasm actors. Satisfies
`Message` bounds trivially.
- **`SharedEngine`** — wraps `Arc<wasmtime::Engine>`. Created once, cloned
cheaply across actors. Sandboxed config: no threads, no SIMD, no reference
types.
- **`WasmActorBuilder`** — takes an engine + raw `.wasm` bytes, compiles the
module, links the `swactor.send` host import, extracts typed function handles,
returns a `WasmActor`.
- **`WasmActor`** — implements `ActorInterface<Incoming = ByteMessage, Response = ()>`.
- **`WasmActorError`** — `MissingExport(&'static str)` or `Wasmtime(wasmtime::Error)`.
---
## 3. Guest ↔ Host Contract
**Guest must export:**
| Export | Signature | Purpose |
|--------|-----------|---------|
| `memory` | WebAssembly linear memory | Host reads/writes message bytes here |
| `alloc` | `(size: i32) -> i32` | Allocate `size` bytes, return pointer |
| `handle` | `(ptr: i32, len: i32)` | Process message at `(ptr, len)` |
**Guest may import:**
| Import | Module | Signature | Purpose |
|--------|--------|-----------|---------|
| `send` | `swactor` | `(dest_ptr: i32, payload_ptr: i32, payload_len: i32)` | Send a message to another actor |
`dest_ptr` points to 32 bytes of `ActorAddress` in guest linear memory.
`payload_ptr` + `payload_len` describe the message bytes.
---
## 4. Handle Cycle (Hot Path)
```
ByteMessage arrives
│
v
1. host calls guest alloc(msg.len) → ptr
│
v
2. host writes msg bytes into guest memory at ptr
│
v
3. host calls guest handle(ptr, len)
│
├── guest may call swactor.send() N times
│ └── each appends (ActorAddress, Vec<u8>) to HostState.outbox
│
v
4. host drains outbox → ctx.send(dest, ByteMessage(payload)) for each
```
Traps during `alloc` or `handle` will panic. Swactor's existing
`catch_unwind` in `tick_all` poisons the actor — consistent with the
panic-safety model.
---
## 5. Guest Modules
Three `#![no_std]` Rust crates compiled to `wasm32-unknown-unknown`:
| Guest | Behavior | Tests it supports |
|-------|----------|-------------------|
| `echo` | Reads 32-byte dest + payload from message; sends payload back to dest | Echo roundtrip, binary preservation |
| `double` | Same framing; sends payload back **twice** | Multi-send verification |
| `silent` | Receives bytes; does nothing | No-output / no-error baseline |
Each guest uses a simple inline bump allocator (64 KiB heap, 8-byte aligned)
and a `#[panic_handler]` that loops. No external dependencies.
Message framing convention: the first 32 bytes of the `ByteMessage` payload
are the destination `ActorAddress`, followed by the actual message bytes.
This allows guests to send replies without hardcoding addresses.
### Building guests
```bash
rustup target add wasm32-unknown-unknown # one-time
cd crates/wasm-actor/tests/guests/echo && cargo build --target wasm32-unknown-unknown --release
cd crates/wasm-actor/tests/guests/double && cargo build --target wasm32-unknown-unknown --release
cd crates/wasm-actor/tests/guests/silent && cargo build --target wasm32-unknown-unknown --release
```
Each guest crate has its own `[workspace]` marker to stay independent of the
root workspace.
---
## 6. Design Decisions & Tradeoffs
| # | Decision | Rationale |
|---|----------|-----------|
| 1 | **wasmtime, not wasmer/wasm3** | Best-maintained, fuel metering support, cranelift JIT |
| 2 | **Raw bytes, not structured messages** | Keeps the boundary simple; framing/serialization is the guest's concern |
| 3 | **Separate crate, not a feature flag** | wasmtime is ~30 crates; most users don't need it in their dependency tree |
| 4 | **Bump allocator in guests** | Zero-dependency, predictable, sufficient for request/response patterns |
| 5 | **Dest address in message payload** | Avoids hardcoded addresses; guests can send to any actor the host tells them about |
| 6 | **Traps = panics (no Result)** | Matches swactor's existing panic-safety model; `catch_unwind` in `tick_all` poisons the actor |
| 7 | **Engine sharing via Arc** | Module compilation is expensive; `SharedEngine` amortizes it across actors |
| 8 | **Maximum sandboxing defaults** | Disabled: threads, SIMD, relaxed SIMD, reference types, multi-value. Enabled: bulk memory (required by most compilers) |
---
## 7. Known Gaps & Future Improvements
| # | Gap | Notes |
|---|-----|-------|
| 1 | **No fuel metering** | wasmtime supports fuel; maps naturally to per-tick actor budgets. Deferred to follow-up. |
| 2 | **No WASI** | No filesystem, network, random, or clock access. Intentional for sandboxing, but limits guest capabilities. |
| 3 | **No guest SDK crate** | The test guests serve as examples. A published `swactor-guest` crate with the alloc/handle/send glue would reduce boilerplate. |
| 4 | **Bump allocator never frees** | Fine for short-lived handle calls, but long-running actors would need a real allocator. |
| 5 | **No pre-compilation cache** | `Module::new()` recompiles every time. wasmtime supports serialized modules for faster cold starts. |
| 6 | **`cargo test -p` doesn't resolve** | Must use `--manifest-path`. Workspace resolution quirk. |
---
## 8. Test Coverage Summary
7 behavioral tests in `crates/wasm-actor/tests/wasm_actor.rs`:
| Test | Scenario |
|------|----------|
| `echo_returns_same_payload` | Send bytes → wasm echoes them back to inbox |
| `echo_preserves_binary_payload` | All 256 byte values survive the roundtrip |
| `silent_produces_no_output` | Guest does nothing; no error, no messages |
| `double_sends_two_copies` | One message in → two messages out |
| `missing_alloc_export_returns_error` | WAT module with no exports → `WasmActorError::MissingExport` |
| `shared_engine_serves_multiple_actors` | Two actors from the same `SharedEngine` work independently |
| `native_actor_communicates_with_wasm_actor` | Native Rust actor → WasmActor → inbox (two-tick delivery) |

115
docs/wasm-actor.md Normal file
View file

@ -0,0 +1,115 @@
# Wasm Actor
The `swactor-wasm-actor` crate runs WebAssembly guest code inside a swactor
actor. The Wasm instance is sandboxed by [wasmtime](https://wasmtime.dev/).
## Architecture
```
┌─ Runtime ──────────────────────────────────────────────────────────────┐
│ │
│ ┌─ WasmActor ──────────────────────────────────────────────────────┐ │
│ │ │ │
│ │ Store<HostState> -- wasmtime store with outbox │ │
│ │ Memory -- guest linear memory │ │
│ │ alloc: TypedFunc -- guest allocator │ │
│ │ handle: TypedFunc -- guest message handler │ │
│ │ │ │
│ │ impl ActorInterface for WasmActor │ │
│ │ Incoming = ByteMessage │ │
│ │ Response = () │ │
│ │ │ │
│ └──────────────────────────────────────────────────────────────────┘ │
│ │
│ ┌─ Native Actors ─────────────────────────────────────────────────┐ │
│ │ (can exchange ByteMessage with WasmActors normally) │ │
│ └─────────────────────────────────────────────────────────────────┘ │
│ │
└────────────────────────────────────────────────────────────────────────┘
```
## Message Flow
```
Host Guest (Wasm)
──── ────────────
ByteMessage arrives
│
├─1─ call alloc(len) ──────────► bump-allocate, return ptr
│
├─2─ write bytes at ptr ───────► (memory updated)
│
├─3─ call handle(ptr, len) ────► process message
│ │
│ ◄── swactor.send() ────────────┤ (0..N times)
│ (buffered in HostState.outbox) │
│ │
├─4─ drain outbox ◄────────────── handle returns
│
v
ctx.send(dest, ByteMessage) for each outbox entry
```
## Guest Contract
Guests are standalone `wasm32-unknown-unknown` modules. They export three
symbols and may import one:
| Direction | Module | Symbol | Signature |
|-----------|--------|--------|-----------|
| **export** | — | `memory` | linear memory |
| **export** | — | `alloc` | `(i32) -> i32` |
| **export** | — | `handle` | `(i32, i32) -> ()` |
| **import** | `swactor` | `send` | `(i32, i32, i32) -> ()` |
The `send` import takes `(dest_ptr, payload_ptr, payload_len)` where
`dest_ptr` points to a 32-byte `ActorAddress` in guest memory.
## Usage
```rust
use swactor::runtime::{Runtime, RuntimeConfig};
use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder};
// Create a shared engine (once)
let engine = SharedEngine::new().unwrap();
// Build an actor from .wasm bytes
let wasm_bytes = std::fs::read("my_guest.wasm").unwrap();
let actor = WasmActorBuilder::new(engine, wasm_bytes)
.build()
.unwrap();
// Use it like any other actor
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(b"hello".to_vec())).unwrap();
rt.tick();
```
## Sandboxing
The `SharedEngine` disables all optional Wasm proposals:
- Threads — disabled
- SIMD / relaxed SIMD — disabled
- Reference types — disabled
- Multi-value — disabled
- Bulk memory — **enabled** (required by most Rust/LLVM toolchains)
No WASI imports are linked. Guests have no access to the filesystem, network,
clock, or random number generator. The only host function available is
`swactor.send`.
## Where Things Live
| File | Purpose |
|------|---------|
| `crates/wasm-actor/src/lib.rs` | `ByteMessage` + re-exports |
| `crates/wasm-actor/src/engine.rs` | `SharedEngine` — sandboxed wasmtime config |
| `crates/wasm-actor/src/builder.rs` | `WasmActorBuilder` — compile, link, instantiate |
| `crates/wasm-actor/src/actor.rs` | `WasmActor` — `ActorInterface` impl |
| `crates/wasm-actor/src/error.rs` | `WasmActorError` |
| `crates/wasm-actor/tests/guests/` | Three test guest crates (echo, double, silent) |
| `crates/wasm-actor/tests/wasm_actor.rs` | 7 integration tests |

View file

@ -2,6 +2,31 @@ use std::any::Any;
use crate::Error; use crate::Error;
/// Why an actor exited.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ExitReason {
/// Actor was explicitly stopped or removed from the pool.
Stopped,
/// Actor panicked during message handling.
Panicked,
/// The node hosting the actor left the cluster (SWIM Dead).
NodeDown,
}
/// Delivered to watchers when a watched actor exits.
///
/// Implements `Message` (Clone + Send + Sync + 'static) so it can be
/// delivered through normal mailbox channels.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ActorExited {
/// The address of the actor that died.
pub addr: ActorAddress,
/// Why it exited.
pub reason: ExitReason,
}
/// The primary trait defining data that can be passed to and from actor processes /// The primary trait defining data that can be passed to and from actor processes
pub trait Message: 'static + Sized + Clone + Send + Sync {} pub trait Message: 'static + Sized + Clone + Send + Sync {}
impl<T: 'static + Sized + Clone + Send + Sync> Message for T {} impl<T: 'static + Sized + Clone + Send + Sync> Message for T {}
@ -32,6 +57,11 @@ pub trait ActorInterface: 'static + Send {
/// If your `Incoming` type IS `Down`, this method is never called — the /// If your `Incoming` type IS `Down`, this method is never called — the
/// normal `handle()` receives the message instead. /// normal `handle()` receives the message instead.
fn handle_down(&mut self, _ctx: &Ctx, _down: Down) {} fn handle_down(&mut self, _ctx: &Ctx, _down: Down) {}
/// Called when a watched actor exits. Override to react to death notifications.
///
/// Default: no-op (notification is silently consumed).
fn on_actor_exit(&mut self, _ctx: &Ctx, _exited: ActorExited) {}
} }
/// A unique address for this actor. 32 bytes is overkill for a small application, /// A unique address for this actor. 32 bytes is overkill for a small application,
@ -106,10 +136,17 @@ where
} }
Err(msg) => msg, Err(msg) => msg,
}; };
match msg.downcast::<Down>() { let msg = match msg.downcast::<Down>() {
Ok(down) => { Ok(down) => {
self.inner.handle_down(ctx, *down); self.inner.handle_down(ctx, *down);
Some("swactor::actor::Down") return Some("swactor::actor::Down");
}
Err(msg) => msg,
};
match msg.downcast::<ActorExited>() {
Ok(exited) => {
self.inner.on_actor_exit(ctx, *exited);
Some("ActorExited")
} }
Err(_) => None, Err(_) => None,
} }
@ -205,6 +242,10 @@ pub trait ContextInner {
fn schedule_timer(&self, request: TimerRequest); fn schedule_timer(&self, request: TimerRequest);
/// Access the runtime extension (if installed). /// Access the runtime extension (if installed).
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>; fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>;
/// Register a watch: watcher receives ActorExited when target dies.
fn watch(&self, watcher: ActorAddress, target: ActorAddress);
/// Cancel a watch.
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress);
} }
/// Actor syscall interface — passed to `ActorInterface::handle()`. /// Actor syscall interface — passed to `ActorInterface::handle()`.
@ -290,4 +331,22 @@ impl<'a> Ctx<'a> {
period, period,
}); });
} }
/// Watch another actor's liveness. If the target dies, this actor
/// receives an `ActorExited` message in its mailbox.
///
/// Watching an already-dead or non-existent actor delivers
/// `ActorExited { reason: Stopped }` on the next tick.
///
/// Calling watch() multiple times on the same target is idempotent —
/// only one notification is delivered.
pub fn watch(&self, target: ActorAddress) {
self.inner.watch(self.self_addr, target);
}
/// Stop watching an actor. No notification will be delivered if the
/// target subsequently dies.
pub fn unwatch(&self, target: ActorAddress) {
self.inner.unwatch(self.self_addr, target);
}
} }

View file

@ -2,13 +2,14 @@ use std::any::Any;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::hash::{BuildHasher, Hasher}; use std::hash::{BuildHasher, Hasher};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock, RwLock}; use std::sync::{Arc, Mutex, OnceLock, RwLock};
use std::thread::Thread; use std::thread::Thread;
use crate::actor::{ActorAddress, AnyActor, Message}; use crate::actor::{ActorAddress, AnyActor, Message};
use crate::channel::Sender; use crate::channel::Sender;
use crate::config::RuntimeConfig; use crate::config::RuntimeConfig;
use crate::stats::WorkerStats; use crate::stats::WorkerStats;
use crate::worker::WatchRegistry;
use crate::Error; use crate::Error;
// ─── Identity Hasher for ActorAddress ─────────────────────────────────────── // ─── Identity Hasher for ActorAddress ───────────────────────────────────────
@ -243,6 +244,7 @@ pub(crate) struct TickContext<'a> {
pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>, pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>,
/// Thread handles for waking parked workers on cross-worker sends. /// Thread handles for waking parked workers on cross-worker sends.
pub(crate) worker_threads: &'a [OnceLock<Thread>], pub(crate) worker_threads: &'a [OnceLock<Thread>],
pub(crate) watch_registry: Option<&'a Arc<Mutex<WatchRegistry>>>,
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>, pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>,
#[cfg(feature = "transport")] #[cfg(feature = "transport")]

View file

@ -1,11 +1,11 @@
use std::any::Any; use std::any::Any;
use std::cell::RefCell; use std::cell::RefCell;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, Mutex, OnceLock};
use std::thread::{self, JoinHandle, Thread}; use std::thread::{self, JoinHandle, Thread};
use std::time::Instant; use std::time::Instant;
use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message, StopSignal, TimerRequest}; use crate::actor::{Actor, ActorAddress, ActorExited, ActorInterface, AnyActor, ExitReason, Message, StopSignal, TimerRequest};
use crate::channel::{Receiver, Sender}; use crate::channel::{Receiver, Sender};
// Re-export config types so existing code using `runtime::RuntimeConfig` still works // Re-export config types so existing code using `runtime::RuntimeConfig` still works
pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig}; pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig};
@ -14,7 +14,7 @@ use crate::extension::RuntimeExtension;
use crate::stats::{StatsHook, WorkerStats}; use crate::stats::{StatsHook, WorkerStats};
// Re-export stats types so existing code using `runtime::*` still works // Re-export stats types so existing code using `runtime::*` still works
pub use crate::stats::{RuntimeStats, WorkerInfo}; pub use crate::stats::{RuntimeStats, WorkerInfo};
use crate::worker::Worker; use crate::worker::{WatchRegistry, Worker};
use crate::Error; use crate::Error;
/// Generic message inbox for receiving messages outside of the runtime. /// Generic message inbox for receiving messages outside of the runtime.
@ -103,6 +103,7 @@ pub struct Runtime {
is_running: AtomicBool, is_running: AtomicBool,
worker_stats: Vec<Arc<WorkerStats>>, worker_stats: Vec<Arc<WorkerStats>>,
stats_hook: Option<Arc<dyn StatsHook>>, stats_hook: Option<Arc<dyn StatsHook>>,
watch_registry: Arc<Mutex<WatchRegistry>>,
/// Workers available for tick(). run() drains this and moves workers to threads. /// Workers available for tick(). run() drains this and moves workers to threads.
tick_workers: RefCell<Vec<Worker>>, tick_workers: RefCell<Vec<Worker>>,
/// Thread handles for waking parked workers. Set by workers on startup via OnceLock. /// Thread handles for waking parked workers. Set by workers on startup via OnceLock.
@ -198,6 +199,7 @@ impl Runtime {
is_running: AtomicBool::new(false), is_running: AtomicBool::new(false),
worker_stats, worker_stats,
stats_hook: None, stats_hook: None,
watch_registry: Arc::new(Mutex::new(WatchRegistry::new())),
tick_workers: RefCell::new(workers), tick_workers: RefCell::new(workers),
worker_threads, worker_threads,
created_at: Instant::now(), created_at: Instant::now(),
@ -304,6 +306,7 @@ impl Runtime {
extension: self.extension.as_deref(), extension: self.extension.as_deref(),
stats_hook: self.stats_hook.as_deref(), stats_hook: self.stats_hook.as_deref(),
worker_threads: &self.worker_threads, worker_threads: &self.worker_threads,
watch_registry: Some(&self.watch_registry),
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
codec_registry: self.codec_registry.as_deref(), codec_registry: self.codec_registry.as_deref(),
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
@ -506,4 +509,25 @@ impl ContextInner for Runtime {
fn extension(&self) -> Option<&dyn RuntimeExtension> { fn extension(&self) -> Option<&dyn RuntimeExtension> {
self.extension.as_deref() self.extension.as_deref()
} }
fn watch(&self, watcher: ActorAddress, target: ActorAddress) {
if self.address_map.lookup(&target).is_some() {
self.watch_registry.lock().unwrap().watch(watcher, target);
} else {
// Target not found — deliver ActorExited immediately.
let msg = ActorExited {
addr: target,
reason: ExitReason::Stopped,
};
// Route to watcher via transfer queue
if let Some(wid) = self.address_map.lookup(&watcher) {
self.transfer_txs[wid.as_usize()]
.send(Envelope::new(watcher, Box::new(msg)));
}
}
}
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) {
self.watch_registry.lock().unwrap().unwatch(watcher, target);
}
} }

View file

@ -1,12 +1,12 @@
use std::any::Any; use std::any::Any;
use std::cell::RefCell; use std::cell::RefCell;
use std::collections::{HashMap, VecDeque}; use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::{Arc, Mutex};
use std::thread; use std::thread;
use std::time::Instant; use std::time::Instant;
use crate::actor::{ActorAddress, AnyActor, CloneMsg, ContextInner, Ctx, StopReason, StopSignal, TimerRequest}; use crate::actor::{ActorAddress, ActorExited, AnyActor, CloneMsg, ContextInner, Ctx, ExitReason, StopReason, StopSignal, TimerRequest};
use crate::channel::Receiver; use crate::channel::Receiver;
use crate::config::MailboxOverflow; use crate::config::MailboxOverflow;
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId}; use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
@ -108,6 +108,95 @@ impl TimerWheel {
} }
} }
// ─── Watch Registry ─────────────────────────────────────────────────────────
/// Tracks watch relationships between actors.
///
/// Shared across workers via `Arc<Mutex<_>>`. Contention is negligible
/// because watch/unwatch operations are rare relative to message sends.
pub(crate) struct WatchRegistry {
/// target → set of watchers awaiting death notification
watchers: HashMap<ActorAddress, HashSet<ActorAddress>>,
/// watcher → set of targets it's watching (reverse index for cleanup)
watching: HashMap<ActorAddress, HashSet<ActorAddress>>,
}
impl WatchRegistry {
pub fn new() -> Self {
Self {
watchers: HashMap::new(),
watching: HashMap::new(),
}
}
pub fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) {
self.watchers.entry(target).or_default().insert(watcher);
self.watching.entry(watcher).or_default().insert(target);
}
pub fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) {
if let Some(set) = self.watchers.get_mut(&target) {
set.remove(&watcher);
if set.is_empty() {
self.watchers.remove(&target);
}
}
if let Some(set) = self.watching.get_mut(&watcher) {
set.remove(&target);
if set.is_empty() {
self.watching.remove(&watcher);
}
}
}
/// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs.
pub fn notify_death(
&mut self,
target: ActorAddress,
reason: ExitReason,
) -> Vec<(ActorAddress, ActorExited)> {
let notification = ActorExited {
addr: target,
reason,
};
let mut result = Vec::new();
if let Some(watcher_set) = self.watchers.remove(&target) {
for watcher in &watcher_set {
result.push((*watcher, notification.clone()));
// clean up reverse index
if let Some(set) = self.watching.get_mut(watcher) {
set.remove(&target);
if set.is_empty() {
self.watching.remove(watcher);
}
}
}
}
result
}
/// Called when a watcher itself dies. Cleans up all its watching entries.
pub fn cleanup_watcher(&mut self, watcher: &ActorAddress) {
if let Some(targets) = self.watching.remove(watcher) {
for target in targets {
if let Some(set) = self.watchers.get_mut(&target) {
set.remove(watcher);
if set.is_empty() {
self.watchers.remove(&target);
}
}
}
}
}
/// Check if a target has any watchers registered.
pub fn has_watchers(&self, target: &ActorAddress) -> bool {
self.watchers.get(target).is_some_and(|s| !s.is_empty())
}
}
// ─── Worker ───────────────────────────────────────────────────────────────── // ─── Worker ─────────────────────────────────────────────────────────────────
/// A worker owns a set of actors and runs them in a loop. /// A worker owns a set of actors and runs them in a loop.
@ -203,6 +292,7 @@ impl Worker {
let timer_requests: RefCell<Vec<TimerRequest>> = RefCell::new(Vec::new()); let timer_requests: RefCell<Vec<TimerRequest>> = RefCell::new(Vec::new());
let processed; let processed;
let deaths;
{ {
let worker_ctx = WorkerContext { let worker_ctx = WorkerContext {
worker_id: self.id, worker_id: self.id,
@ -212,7 +302,7 @@ impl Worker {
timer_requests: &timer_requests, timer_requests: &timer_requests,
stats: &self.stats, stats: &self.stats,
}; };
processed = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests); (processed, deaths) = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests);
if processed > 0 { if processed > 0 {
did_work = true; did_work = true;
} }
@ -256,6 +346,40 @@ impl Worker {
} }
} }
} }
// 5b. Process actor deaths → deliver ActorExited to watchers
if !deaths.is_empty() {
did_work = true;
if let Some(registry) = &tc.watch_registry {
let mut reg = registry.lock().unwrap();
for (dead_addr, reason) in deaths {
let notifications = reg.notify_death(dead_addr, reason);
for (watcher_addr, msg) in notifications {
// Deliver ActorExited as a normal message via the address map
match tc.address_map.lookup(&watcher_addr) {
Some(wid) if wid == self.id => {
self.pool.deliver(&watcher_addr, Box::new(msg));
}
Some(wid) => {
tc.transfer_txs[wid.as_usize()]
.send(Envelope::new(watcher_addr, Box::new(msg)));
}
None => {
// Watcher not in address map — may be an inbox or remote.
// Try inbox registry as best effort.
let _ = tc.inbox_registry.try_deliver(
watcher_addr,
Box::new(msg),
);
}
}
}
// Clean up the dead actor's own watches (things it was watching)
reg.cleanup_watcher(&dead_addr);
}
}
}
let t5 = Instant::now(); let t5 = Instant::now();
// 6. Publish stats (skip entirely when idle to avoid allocation + mutex) // 6. Publish stats (skip entirely when idle to avoid allocation + mutex)
@ -451,6 +575,29 @@ impl ContextInner for WorkerContext<'_> {
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> { fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
self.tc.extension self.tc.extension
} }
fn watch(&self, watcher: ActorAddress, target: ActorAddress) {
if let Some(registry) = &self.tc.watch_registry {
// Check if target exists in the address map
if self.tc.address_map.lookup(&target).is_some() {
registry.lock().unwrap().watch(watcher, target);
} else {
// Target not found — deliver ActorExited { reason: Stopped } immediately.
// Buffer in pending_local so it arrives on next tick.
let msg = ActorExited {
addr: target,
reason: ExitReason::Stopped,
};
self.pending_local.borrow_mut().push((watcher, Box::new(msg)));
}
}
}
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) {
if let Some(registry) = &self.tc.watch_registry {
registry.lock().unwrap().unwatch(watcher, target);
}
}
} }
struct ActorSlot { struct ActorSlot {
@ -531,7 +678,7 @@ impl ActorPool {
std::mem::replace(&mut self.drops_this_tick, 0) std::mem::replace(&mut self.drops_this_tick, 0)
} }
/// Tick all actors in the pool. Returns the number of messages processed. /// Tick all actors in the pool. Returns (messages_processed, newly_dead_actors).
/// ///
/// Each actor processes up to `budget` messages per tick (0 = unlimited). /// Each actor processes up to `budget` messages per tick (0 = unlimited).
/// This prevents a single hot actor from starving others on the same worker. /// This prevents a single hot actor from starving others on the same worker.
@ -541,8 +688,9 @@ impl ActorPool {
stats: &WorkerStats, stats: &WorkerStats,
budget: usize, budget: usize,
stop_requests: &RefCell<Vec<ActorAddress>>, stop_requests: &RefCell<Vec<ActorAddress>>,
) -> usize { ) -> (usize, Vec<(ActorAddress, ExitReason)>) {
let mut count = 0; let mut count = 0;
let mut deaths = Vec::new();
for (&addr, slot) in self.actors.iter_mut() { for (&addr, slot) in self.actors.iter_mut() {
if slot.poisoned || slot.stopping { if slot.poisoned || slot.stopping {
// Discard all messages for poisoned/stopping actors // Discard all messages for poisoned/stopping actors
@ -606,6 +754,8 @@ impl ActorPool {
#[cfg(feature = "tracing")] #[cfg(feature = "tracing")]
tracing::error!(actor_addr = %addr, "actor.panicked"); tracing::error!(actor_addr = %addr, "actor.panicked");
slot.poisoned = true; slot.poisoned = true;
slot.mailbox.clear();
deaths.push((addr, ExitReason::Panicked));
break; break;
} }
Ok(Some(type_name)) => { Ok(Some(type_name)) => {
@ -624,6 +774,7 @@ impl ActorPool {
slot.stopping = true; slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed); stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear(); slot.mailbox.clear();
deaths.push((addr, ExitReason::Stopped));
break; break;
} }
} }
@ -633,7 +784,7 @@ impl ActorPool {
} }
} }
} }
count (count, deaths)
} }
pub fn len(&self) -> usize { pub fn len(&self) -> usize {

378
tests/watch_api.rs Normal file
View file

@ -0,0 +1,378 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use swactor::actor::{ActorAddress, ActorExited, ActorInterface, ExitReason};
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
// ── Actors ──────────────────────────────────────────────────────────────────
/// An actor that panics when it receives PanicMsg.
struct PanicOnCommand;
#[derive(Clone)]
struct PanicMsg;
impl ActorInterface for PanicOnCommand {
type Incoming = PanicMsg;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: PanicMsg) {
panic!("deliberate panic for test");
}
}
/// An actor that watches targets and counts exit notifications.
struct ExitWatcher {
exit_count: Arc<AtomicUsize>,
last_reason: Arc<std::sync::Mutex<Option<ExitReason>>>,
last_addr: Arc<std::sync::Mutex<Option<ActorAddress>>>,
}
#[derive(Clone)]
enum WatcherCmd {
WatchThis(ActorAddress),
UnwatchThis(ActorAddress),
}
impl ActorInterface for ExitWatcher {
type Incoming = WatcherCmd;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: WatcherCmd) {
match msg {
WatcherCmd::WatchThis(target) => {
ctx.watch(target);
}
WatcherCmd::UnwatchThis(target) => {
ctx.unwatch(target);
}
}
}
fn on_actor_exit(&mut self, _ctx: &Ctx, exited: ActorExited) {
self.exit_count.fetch_add(1, Ordering::SeqCst);
*self.last_reason.lock().unwrap() = Some(exited.reason);
*self.last_addr.lock().unwrap() = Some(exited.addr);
}
}
impl ExitWatcher {
fn new() -> (Self, WatcherState) {
let exit_count = Arc::new(AtomicUsize::new(0));
let last_reason = Arc::new(std::sync::Mutex::new(None));
let last_addr = Arc::new(std::sync::Mutex::new(None));
let state = WatcherState {
exit_count: exit_count.clone(),
last_reason: last_reason.clone(),
last_addr: last_addr.clone(),
};
(
ExitWatcher {
exit_count,
last_reason,
last_addr,
},
state,
)
}
}
/// Shared state for inspecting what ExitWatcher observed.
struct WatcherState {
exit_count: Arc<AtomicUsize>,
last_reason: Arc<std::sync::Mutex<Option<ExitReason>>>,
last_addr: Arc<std::sync::Mutex<Option<ActorAddress>>>,
}
impl WatcherState {
fn count(&self) -> usize {
self.exit_count.load(Ordering::SeqCst)
}
fn last_reason(&self) -> Option<ExitReason> {
self.last_reason.lock().unwrap().clone()
}
fn last_addr(&self) -> Option<ActorAddress> {
*self.last_addr.lock().unwrap()
}
}
/// A silent actor that does nothing (for targets that shouldn't panic).
struct Sleeper;
#[derive(Clone)]
struct Noop;
impl ActorInterface for Sleeper {
type Incoming = Noop;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: Noop) {}
}
// ── Helper ──────────────────────────────────────────────────────────────────
fn tick_n(rt: &Runtime, n: usize) {
for _ in 0..n {
rt.tick();
}
}
fn single_thread_config() -> RuntimeConfig {
RuntimeConfig {
num_threads: 1,
..RuntimeConfig::default()
}
}
// ── Tests ───────────────────────────────────────────────────────────────────
/// Given a watcher and a target actor,
/// when the target panics,
/// then the watcher's on_actor_exit fires with ExitReason::Panicked.
#[test]
fn watch_receives_notification_on_panic() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
// Tell watcher to watch the target
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill the target
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "watcher should have received exactly one ActorExited");
assert_eq!(state.last_reason(), Some(ExitReason::Panicked));
assert_eq!(state.last_addr(), Some(target));
}
/// Given a watcher that watches then unwatches a target,
/// when the target panics,
/// then the watcher receives NO notification.
#[test]
fn unwatch_prevents_notification() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
// Watch
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Unwatch
rt.send_to(watcher, WatcherCmd::UnwatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill target
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 0, "after unwatch, no notification should be delivered");
}
/// Given a watch on an address that was never spawned,
/// then the watcher receives ActorExited { reason: Stopped }.
#[test]
fn watch_nonexistent_actor_delivers_stopped() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let watcher = rt.spawn(watcher_actor).unwrap();
let nonexistent = ActorAddress::new_random();
rt.send_to(watcher, WatcherCmd::WatchThis(nonexistent)).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "should receive ActorExited for non-existent target");
assert_eq!(state.last_reason(), Some(ExitReason::Stopped));
assert_eq!(state.last_addr(), Some(nonexistent));
}
/// Given a watcher that dies before the target,
/// when the target subsequently panics,
/// then there is no panic or leak.
#[test]
fn watcher_dies_before_target_no_panic() {
let rt = Runtime::new(single_thread_config());
let target = rt.spawn(PanicOnCommand).unwrap();
let (watcher_actor, _state) = ExitWatcher::new();
let watcher = rt.spawn(watcher_actor).unwrap();
// Watch
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill the watcher first (send it a type-mismatched panic msg directly)
// Actually, ExitWatcher doesn't panic. Use Runtime-level watch + PanicOnCommand.
let rt2 = Runtime::new(single_thread_config());
let target2 = rt2.spawn(PanicOnCommand).unwrap();
let watcher2 = rt2.spawn(PanicOnCommand).unwrap();
use swactor::actor::ContextInner;
rt2.watch(watcher2, target2);
tick_n(&rt2, 3);
// Kill watcher first
rt2.send_to(watcher2, PanicMsg).unwrap();
tick_n(&rt2, 5);
// Kill target — should not crash
rt2.send_to(target2, PanicMsg).unwrap();
tick_n(&rt2, 5);
// If we got here, no crash.
}
/// Given a watcher that calls watch() twice on the same target,
/// when the target panics,
/// then the watcher receives exactly one notification.
#[test]
fn idempotent_watch_delivers_one_notification() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
// Watch twice
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill target
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "double watch should produce exactly one notification");
}
/// Given multiple watchers on the same target,
/// when the target panics,
/// then all watchers receive the notification.
#[test]
fn multiple_watchers_all_notified() {
let rt = Runtime::new(single_thread_config());
let (w1_actor, s1) = ExitWatcher::new();
let (w2_actor, s2) = ExitWatcher::new();
let (w3_actor, s3) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let w1 = rt.spawn(w1_actor).unwrap();
let w2 = rt.spawn(w2_actor).unwrap();
let w3 = rt.spawn(w3_actor).unwrap();
rt.send_to(w1, WatcherCmd::WatchThis(target)).unwrap();
rt.send_to(w2, WatcherCmd::WatchThis(target)).unwrap();
rt.send_to(w3, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(s1.count(), 1, "watcher 1 should be notified");
assert_eq!(s2.count(), 1, "watcher 2 should be notified");
assert_eq!(s3.count(), 1, "watcher 3 should be notified");
}
/// Self-watch doesn't crash the runtime.
#[test]
fn self_watch_does_not_crash() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, _state) = ExitWatcher::new();
let actor = rt.spawn(watcher_actor).unwrap();
rt.send_to(actor, WatcherCmd::WatchThis(actor)).unwrap();
tick_n(&rt, 5);
// No crash = pass
}
/// Runtime-level watch (outside actor context) delivers notification.
#[test]
fn runtime_level_watch_delivers_notification() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
tick_n(&rt, 2); // ensure both spawned
use swactor::actor::ContextInner;
rt.watch(watcher, target);
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "runtime-level watch should deliver notification");
assert_eq!(state.last_reason(), Some(ExitReason::Panicked));
}
/// Runtime-level watch on non-existent address delivers Stopped.
#[test]
fn runtime_level_watch_nonexistent_delivers_stopped() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let watcher = rt.spawn(watcher_actor).unwrap();
tick_n(&rt, 2);
let fake = ActorAddress::new_random();
use swactor::actor::ContextInner;
rt.watch(watcher, fake);
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "watching non-existent from runtime should deliver Stopped");
assert_eq!(state.last_reason(), Some(ExitReason::Stopped));
}
/// Given a watcher watching target via on_actor_exit,
/// when target panics,
/// then the watcher can react by spawning a replacement (supervision pattern).
#[test]
fn watcher_can_react_to_death_by_spawning() {
let rt = Runtime::new(single_thread_config());
let spawned = Arc::new(AtomicUsize::new(0));
struct Supervisor {
spawned_count: Arc<AtomicUsize>,
}
#[derive(Clone)]
enum SupervisorMsg {
WatchThis(ActorAddress),
}
impl ActorInterface for Supervisor {
type Incoming = SupervisorMsg;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) {
match msg {
SupervisorMsg::WatchThis(target) => ctx.watch(target),
}
}
fn on_actor_exit(&mut self, ctx: &Ctx, _exited: ActorExited) {
// React: spawn a replacement
let replacement = ctx.spawn(Sleeper).unwrap();
let _ = replacement;
self.spawned_count.fetch_add(1, Ordering::SeqCst);
}
}
let target = rt.spawn(PanicOnCommand).unwrap();
let sup = rt.spawn(Supervisor { spawned_count: spawned.clone() }).unwrap();
rt.send_to(sup, SupervisorMsg::WatchThis(target)).unwrap();
tick_n(&rt, 3);
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(spawned.load(Ordering::SeqCst), 1, "supervisor should have spawned a replacement");
}