Compare commits

...

4 commits

Author SHA1 Message Date
Developer
669599a9e7 feat: cluster registry — gossip-propagated naming via LWW-Register CRDT
Add a ClusterRegistry that maps human-readable names to (ActorAddress, NodeId)
pairs, replicated across the cluster via SWIM gossip piggyback. Uses
last-writer-wins semantics with timestamp > generation > node_id tie-breaking,
and tombstone-based unregistration with configurable GC.

Key design: DistributedNode intercepts piggyback bytes on all outgoing/incoming
SWIM messages, wrapping them in a PiggybackPayload that carries both membership
updates and registry entries. SwimNode and DisseminationQueue remain untouched.

- New: crates/distribution/src/registry.rs (ClusterRegistry, pack/unpack helpers)
- Modified: node.rs (register_name/unregister_name/resolve_name API, piggyback
  interception, node death tombstoning)
- Modified: snapshot.rs (RegistryEntryInfo in DistributionNodeSnapshot)
- 12 behavioral tests in crates/distribution/tests/registry.rs
- Remove CLAUDE/ from git tracking, add to .gitignore

Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
2026-02-13 07:34:44 +00:00
Developer
bf79d63be4 feat: command interface — frontend-agnostic dispatch crate
Add swactor-command crate that decouples command dispatch from the
dashboard, enabling the same commands from REPL, REST, TUI, or any
future frontend.

- CommandRouter with dispatch(), CommandHandler trait, CommandMeta
- CommandRequest/CommandResponse (serde-serializable JSON protocol)
- CommandContext with Arc<Runtime> + optional StatsEnricher trait
- 9 built-in commands extracted from investigate.rs:
  help, overview, workers, worker, actors, actor, hot, phases, diff,
  shutdown
- REPL line parser (parse_line) with positional arg mapping
- REST adapter (from_query_params) for HTTP query parameters
- Dashboard integration: StatsEnricher impl for StatsCollector,
  investigate.rs refactored to thin CommandRouter wrapper,
  server.rs updated to share CommandRouter across handler threads
- 18 behavioral tests in crates/command/tests/command_api.rs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 07:25:50 +00:00
Developer
473999d1df feat: actor watching — local death notifications
Add watch/unwatch API to the actor system so actors can monitor each
other's liveness. When a watched actor dies (panic or stop), watchers
receive an ActorExited notification via on_actor_exit().

- ExitReason enum (Stopped, Panicked, NodeDown) and ActorExited struct
- ContextInner::watch()/unwatch() + Ctx typed wrappers
- ActorInterface::on_actor_exit() default method (system message fallback)
- WatchRegistry in worker with bidirectional tracking
- Death notification dispatch as phase 5b in tick_once
- Runtime-level watch for external callers
- 10 behavioral tests in tests/watch_api.rs
- Design documents for OS features in docs/os-design/

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 07:25:27 +00:00
Developer
60ee2d1a2c feat: wasm actor crate — run WebAssembly guests inside swactor actors
Adds `crates/wasm-actor/` (swactor-wasm-actor), which embeds wasmtime-sandboxed
Wasm instances inside regular swactor actors. Messages flow as raw bytes through
the guest ↔ host contract (alloc/handle exports, swactor.send import). The host
drains an outbox after each handle call and routes messages via ctx.send().

Includes 3 no_std guest modules (echo, double, silent) and 7 integration tests
covering roundtrip delivery, binary fidelity, multi-send, error cases, engine
sharing, and native↔wasm interop.

Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski
2026-02-13 07:19:09 +00:00
49 changed files with 6825 additions and 644 deletions

4
.gitignore vendored
View file

@ -11,3 +11,7 @@ corpus
**/deps.html **/deps.html
docs/architecture.dot docs/architecture.dot
docs/architecture.html docs/architecture.html
# Claude session files
CLAUDE/
.claude/

1083
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,5 +1,5 @@
[workspace] [workspace]
members = [".", "crates/python", "crates/wasm", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std"] members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std", "crates/command"]
exclude = ["tools/depgraph"] exclude = ["tools/depgraph"]
[package] [package]

View file

@ -0,0 +1,9 @@
[package]
name = "swactor-command"
version = "0.1.0"
edition = "2024"
[dependencies]
swactor = { path = "../..", features = ["serde"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"

View file

@ -0,0 +1,561 @@
//! Built-in command handlers for runtime inspection and management.
//!
//! Extracted from `crates/runtime-dashboard/src/investigate.rs`.
use std::collections::HashMap;
use std::time::{Duration, Instant};
use swactor::actor::ActorAddress;
use swactor::stats::TickTiming;
use crate::{CommandContext, CommandHandler, CommandMeta, CommandResponse};
// ─── Arg helpers ─────────────────────────────────────────────────────────────
fn arg_str<'a>(args: &'a HashMap<String, serde_json::Value>, key: &str) -> Option<&'a str> {
args.get(key).and_then(|v| v.as_str())
}
fn arg_usize(args: &HashMap<String, serde_json::Value>, key: &str) -> Option<usize> {
args.get(key).and_then(|v| {
v.as_u64()
.map(|n| n as usize)
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
})
}
fn arg_f64(args: &HashMap<String, serde_json::Value>, key: &str) -> Option<f64> {
args.get(key).and_then(|v| {
v.as_f64()
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
})
}
// ─── Display helpers ─────────────────────────────────────────────────────────
fn format_addr(addr: &ActorAddress) -> String {
format!("{addr}")
}
fn full_hex(addr: &ActorAddress) -> String {
addr.0.iter().map(|b| format!("{b:02x}")).collect()
}
// ─── Phase breakdown helper ──────────────────────────────────────────────────
fn compute_phase_breakdown(timings: &[TickTiming]) -> serde_json::Value {
if timings.is_empty() {
return serde_json::json!({
"ticks": 0,
"active_pct": 0.0,
"avg_tick_us": 0.0,
"phases_us": [0, 0, 0, 0, 0, 0],
"phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
});
}
let n = timings.len();
let active = timings.iter().filter(|t| t.did_work).count();
let active_pct = (active as f64 / n as f64) * 100.0;
let mut phase_sums = [0u64; 6];
for t in timings {
for (i, &us) in t.phase_us.iter().enumerate() {
phase_sums[i] += us;
}
}
let total_us: u64 = phase_sums.iter().sum();
let avg_tick_us = total_us as f64 / n as f64;
let phases_pct: Vec<f64> = if total_us == 0 {
vec![0.0; 6]
} else {
phase_sums
.iter()
.map(|&s| (s as f64 / total_us as f64) * 100.0)
.collect()
};
serde_json::json!({
"ticks": n,
"active_pct": active_pct,
"avg_tick_us": avg_tick_us,
"phases_us": phase_sums,
"phases_pct": phases_pct,
})
}
// ─── Read Commands ───────────────────────────────────────────────────────────
pub struct OverviewCommand;
impl CommandHandler for OverviewCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "overview",
description: "Summary: worker count, actor count, total messages, mailbox depth, panics",
usage: "overview",
is_write: false,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.enriched_stats();
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum();
let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum();
let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum();
let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum();
let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum();
let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum();
CommandResponse::ok(
"overview",
serde_json::json!({
"workers": stats.num_workers,
"actors": stats.actor_details.len(),
"total_messages_processed": total_msgs,
"total_mailbox_depth": total_mailbox,
"total_panics": total_panics,
"total_type_mismatches": total_type_mismatches,
"sends": {
"local": total_local,
"cross_worker": total_cross,
"inbox": total_inbox,
},
}),
)
}
}
pub struct WorkersCommand;
impl CommandHandler for WorkersCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "workers",
description: "Per-worker stats: actors, mailbox depth, messages, sends, panics",
usage: "workers",
is_write: false,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.stats();
let workers: Vec<_> = stats
.workers
.iter()
.map(|w| {
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
})
})
.collect();
CommandResponse::ok("workers", workers)
}
}
pub struct WorkerCommand;
impl CommandHandler for WorkerCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "worker",
description: "Single worker detail with tick-phase timing breakdown",
usage: "worker <id>",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let id = match arg_usize(args, "id") {
Some(id) => id,
None => return CommandResponse::err("worker", "usage: worker <id>"),
};
let stats = ctx.enriched_stats();
let w = match stats.workers.iter().find(|w| w.id == id) {
Some(w) => w,
None => {
return CommandResponse::err(
"worker",
format!("worker {id} not found (have 0..{})", stats.num_workers),
)
}
};
let timings = stats.tick_timings.get(id).cloned().unwrap_or_default();
let phase_breakdown = compute_phase_breakdown(&timings);
let actors_on_worker: Vec<_> = stats
.actor_details
.iter()
.filter(|a| a.worker_id == id)
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok(
"worker",
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
"tick_phases": phase_breakdown,
"actor_details": actors_on_worker,
}),
)
}
}
pub struct ActorsCommand;
impl CommandHandler for ActorsCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "actors",
description: "List actors with optional sorting, limit, and worker filter",
usage: "actors [--sort mailbox|worker|address] [--limit N] [--worker W]",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.enriched_stats();
let mut actors = stats.actor_details.clone();
let sort_by = arg_str(args, "sort").unwrap_or("mailbox");
let limit = arg_usize(args, "limit").unwrap_or(usize::MAX);
let worker_filter = arg_usize(args, "worker");
if let Some(wid) = worker_filter {
actors.retain(|a| a.worker_id == wid);
}
match sort_by {
"mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)),
"worker" => actors.sort_by_key(|a| a.worker_id),
"address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)),
other => {
return CommandResponse::err(
"actors",
format!("unknown sort field `{other}` — use mailbox|worker|address"),
)
}
}
actors.truncate(limit);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok(
"actors",
serde_json::json!({
"total": stats.actor_details.len(),
"returned": rows.len(),
"sort": sort_by,
"actors": rows,
}),
)
}
}
pub struct ActorCommand;
impl CommandHandler for ActorCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "actor",
description: "Find actor(s) whose address starts with the given hex prefix",
usage: "actor <hex_prefix>",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let prefix = match arg_str(args, "prefix") {
Some(p) => p,
None => return CommandResponse::err("actor", "usage: actor <hex_prefix>"),
};
let stats = ctx.enriched_stats();
let matches: Vec<_> = stats
.actor_details
.iter()
.filter(|a| full_hex(&a.address).starts_with(prefix))
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok(
"actor",
serde_json::json!({
"prefix": prefix,
"matches": matches.len(),
"actors": matches,
}),
)
}
}
pub struct HotCommand;
impl CommandHandler for HotCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "hot",
description: "Top N actors by mailbox depth (default 10)",
usage: "hot [N]",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let n = arg_usize(args, "n").unwrap_or(10);
let stats = ctx.enriched_stats();
let mut actors = stats.actor_details.clone();
actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth));
actors.truncate(n);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
CommandResponse::ok("hot", rows)
}
}
pub struct PhasesCommand;
impl CommandHandler for PhasesCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "phases",
description: "Tick-phase time breakdown (all workers or one)",
usage: "phases [worker_id]",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let stats = ctx.stats();
let worker_filter = arg_usize(args, "worker");
let phase_names = [
"spawn_drain",
"transfer_drain",
"tick_all",
"spawn_drain_2",
"pending_local",
"stats_publish",
];
let mut results = Vec::new();
for (i, timings) in stats.tick_timings.iter().enumerate() {
if let Some(wid) = worker_filter {
if i != wid {
continue;
}
}
let breakdown = compute_phase_breakdown(timings);
results.push(serde_json::json!({
"worker_id": i,
"ticks_sampled": timings.len(),
"phases": breakdown,
"phase_names": phase_names,
}));
}
CommandResponse::ok("phases", results)
}
}
pub struct DiffCommand;
impl CommandHandler for DiffCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "diff",
description: "Collect two snapshots N seconds apart, report deltas and rates",
usage: "diff <seconds>",
is_write: false,
}
}
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
let secs = match arg_f64(args, "seconds") {
Some(s) if s > 0.0 && s <= 30.0 => s,
Some(_) => return CommandResponse::err("diff", "seconds must be between 0 and 30"),
None => return CommandResponse::err("diff", "usage: diff <seconds>"),
};
let before = ctx.enriched_stats();
let t0 = Instant::now();
std::thread::sleep(Duration::from_secs_f64(secs));
let after = ctx.enriched_stats();
let elapsed = t0.elapsed().as_secs_f64();
let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum();
let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum();
let delta_msgs = msgs_after.saturating_sub(msgs_before);
let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum();
let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum();
let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum();
let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum();
let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum();
let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum();
let per_worker: Vec<_> = after
.workers
.iter()
.enumerate()
.map(|(i, w)| {
let prev = before.workers.get(i);
let d = prev
.map(|p| w.messages_processed.saturating_sub(p.messages_processed))
.unwrap_or(0);
serde_json::json!({
"worker_id": i,
"delta_messages": d,
"msg_per_sec": d as f64 / elapsed,
"actors_before": prev.map(|p| p.num_actors).unwrap_or(0),
"actors_after": w.num_actors,
"mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0),
"mailbox_after": w.mailbox_depth,
})
})
.collect();
CommandResponse::ok(
"diff",
serde_json::json!({
"elapsed_s": elapsed,
"actors_before": before.actor_details.len(),
"actors_after": after.actor_details.len(),
"delta_messages": delta_msgs,
"msg_per_sec": delta_msgs as f64 / elapsed,
"delta_local_sends": local_after.saturating_sub(local_before),
"delta_cross_sends": cross_after.saturating_sub(cross_before),
"mailbox_before": mailbox_before,
"mailbox_after": mailbox_after,
"per_worker": per_worker,
}),
)
}
}
// ─── Write Commands ──────────────────────────────────────────────────────────
pub struct ShutdownCommand;
impl CommandHandler for ShutdownCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "shutdown",
description: "Signal the runtime to shut down gracefully",
usage: "shutdown",
is_write: true,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse {
ctx.runtime.shutdown();
CommandResponse::ok(
"shutdown",
serde_json::json!({"status": "shutdown signaled"}),
)
}
}

242
crates/command/src/lib.rs Normal file
View file

@ -0,0 +1,242 @@
//! Frontend-agnostic command dispatch for swactor runtimes.
//!
//! Provides [`CommandRouter`] that maps command names to [`CommandHandler`]
//! implementations, with built-in commands for runtime inspection and management.
//!
//! # Architecture
//!
//! ```text
//! Frontend (REPL, REST, TUI, WebSocket)
//! │
//! ▼
//! CommandRouter::dispatch(CommandRequest, CommandContext)
//! │
//! ├── built-in handlers (overview, workers, actors, …)
//! └── custom handlers (user-registered)
//! ```
pub mod builtins;
mod parse;
pub use parse::{from_query_params, parse_line};
use std::collections::HashMap;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use swactor::runtime::Runtime;
use swactor::stats::RuntimeStats;
// ─── Core Types ──────────────────────────────────────────────────────────────
/// A command request.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandRequest {
pub command: String,
pub args: HashMap<String, serde_json::Value>,
}
/// A command response. Always JSON-serializable.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandResponse {
pub ok: bool,
pub command: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub data: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
impl CommandResponse {
pub fn ok(command: &str, data: impl Serialize) -> Self {
Self {
ok: true,
command: command.to_string(),
data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)),
error: None,
}
}
pub fn err(command: &str, msg: impl Into<String>) -> Self {
Self {
ok: false,
command: command.to_string(),
data: None,
error: Some(msg.into()),
}
}
/// Serialize to a single JSON line (for REPL/wire protocol).
pub fn to_json_line(&self) -> String {
serde_json::to_string(self).unwrap_or_else(|e| {
format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#)
})
}
}
// ─── Handler Trait ───────────────────────────────────────────────────────────
/// Metadata about a command, used for help text and validation.
pub struct CommandMeta {
pub name: &'static str,
pub description: &'static str,
pub usage: &'static str,
pub is_write: bool,
}
/// A command handler. Implementations are stateless — all state
/// comes through [`CommandContext`].
pub trait CommandHandler: Send + Sync {
fn meta(&self) -> CommandMeta;
fn handle(
&self,
args: &HashMap<String, serde_json::Value>,
ctx: &CommandContext,
) -> CommandResponse;
}
// ─── Stats Enrichment ────────────────────────────────────────────────────────
/// Enriches [`RuntimeStats`] with per-actor detail.
///
/// Implement this on your stats collector so command handlers can access
/// enriched data without depending on the dashboard crate.
pub trait StatsEnricher: Send + Sync {
fn enrich(&self, stats: &mut RuntimeStats);
}
// ─── Context ─────────────────────────────────────────────────────────────────
/// Context available to command handlers.
pub struct CommandContext {
pub runtime: Arc<Runtime>,
pub enricher: Option<Arc<dyn StatsEnricher>>,
}
impl CommandContext {
pub fn new(runtime: Arc<Runtime>) -> Self {
Self {
runtime,
enricher: None,
}
}
pub fn with_enricher(
runtime: Arc<Runtime>,
enricher: Arc<dyn StatsEnricher>,
) -> Self {
Self {
runtime,
enricher: Some(enricher),
}
}
/// Raw runtime stats (no enrichment).
pub fn stats(&self) -> RuntimeStats {
self.runtime.stats()
}
/// Runtime stats enriched with per-actor detail (if an enricher is set).
pub fn enriched_stats(&self) -> RuntimeStats {
let mut s = self.runtime.stats();
if let Some(e) = &self.enricher {
e.enrich(&mut s);
}
s
}
}
// ─── Router ──────────────────────────────────────────────────────────────────
/// Central command dispatch.
pub struct CommandRouter {
handlers: HashMap<String, Box<dyn CommandHandler>>,
}
impl CommandRouter {
pub fn new() -> Self {
Self {
handlers: HashMap::new(),
}
}
/// Create a router with all built-in commands registered.
pub fn with_builtins() -> Self {
let mut router = Self::new();
router.register(Box::new(builtins::OverviewCommand));
router.register(Box::new(builtins::WorkersCommand));
router.register(Box::new(builtins::WorkerCommand));
router.register(Box::new(builtins::ActorsCommand));
router.register(Box::new(builtins::ActorCommand));
router.register(Box::new(builtins::HotCommand));
router.register(Box::new(builtins::PhasesCommand));
router.register(Box::new(builtins::DiffCommand));
router.register(Box::new(builtins::ShutdownCommand));
router
}
/// Register a custom command handler.
pub fn register(&mut self, handler: Box<dyn CommandHandler>) {
let name = handler.meta().name.to_string();
self.handlers.insert(name, handler);
}
/// Dispatch a command request.
///
/// The `help` command is handled directly by the router (it needs
/// access to all registered handlers).
pub fn dispatch(
&self,
req: &CommandRequest,
ctx: &CommandContext,
) -> CommandResponse {
if req.command == "help" {
return self.cmd_help();
}
match self.handlers.get(&req.command) {
Some(handler) => handler.handle(&req.args, ctx),
None => CommandResponse::err(
&req.command,
format!("unknown command `{}` — try `help`", req.command),
),
}
}
fn cmd_help(&self) -> CommandResponse {
let mut commands: Vec<serde_json::Value> = self
.handlers
.values()
.map(|h| {
let m = h.meta();
serde_json::json!({
"name": m.name,
"usage": m.usage,
"description": m.description,
"is_write": m.is_write,
})
})
.collect();
// Add help itself
commands.push(serde_json::json!({
"name": "help",
"usage": "help",
"description": "List all available commands",
"is_write": false,
}));
commands.sort_by(|a, b| {
a["name"]
.as_str()
.unwrap_or("")
.cmp(b["name"].as_str().unwrap_or(""))
});
CommandResponse::ok("help", serde_json::json!({ "commands": commands }))
}
/// List names of all registered commands (sorted).
pub fn command_names(&self) -> Vec<&str> {
let mut names: Vec<_> = self.handlers.keys().map(|s| s.as_str()).collect();
names.push("help");
names.sort();
names
}
}

View file

@ -0,0 +1,90 @@
//! Input parsers for REPL lines and HTTP query parameters.
use std::collections::HashMap;
use crate::CommandRequest;
/// Parse a REPL text line into a [`CommandRequest`].
///
/// Handles `--flag value` pairs and maps positional arguments to
/// command-specific named parameters.
///
/// # Examples
///
/// ```text
/// "overview" → { command: "overview", args: {} }
/// "worker 3" → { command: "worker", args: { "id": "3" } }
/// "actors --sort mailbox" → { command: "actors", args: { "sort": "mailbox" } }
/// "hot 5" → { command: "hot", args: { "n": "5" } }
/// ```
pub fn parse_line(line: &str) -> CommandRequest {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.is_empty() {
return CommandRequest {
command: "help".to_string(),
args: HashMap::new(),
};
}
let command = parts[0].to_string();
let rest = &parts[1..];
let mut args = HashMap::new();
let mut i = 0;
let mut positional = 0;
while i < rest.len() {
if let Some(key) = rest[i].strip_prefix("--") {
if i + 1 < rest.len() && !rest[i + 1].starts_with("--") {
args.insert(
key.to_string(),
serde_json::Value::String(rest[i + 1].to_string()),
);
i += 2;
} else {
args.insert(key.to_string(), serde_json::Value::Bool(true));
i += 1;
}
} else {
let name = positional_arg_name(&command, positional);
if !name.is_empty() {
args.insert(
name.to_string(),
serde_json::Value::String(rest[i].to_string()),
);
}
positional += 1;
i += 1;
}
}
CommandRequest { command, args }
}
/// Convert HTTP query parameters to a [`CommandRequest`].
///
/// The `cmd` parameter becomes the command name; all other parameters
/// become string-valued arguments.
pub fn from_query_params(params: &HashMap<String, String>) -> CommandRequest {
let command = params
.get("cmd")
.cloned()
.unwrap_or_else(|| "help".into());
let args: HashMap<String, serde_json::Value> = params
.iter()
.filter(|(k, _)| *k != "cmd")
.map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone())))
.collect();
CommandRequest { command, args }
}
/// Map positional argument index to the named parameter for each command.
fn positional_arg_name(command: &str, position: usize) -> &'static str {
match (command, position) {
("worker", 0) => "id",
("actor", 0) => "prefix",
("hot", 0) => "n",
("phases", 0) => "worker",
("diff", 0) => "seconds",
_ => "",
}
}

View file

@ -0,0 +1,367 @@
//! Behavioral tests for the swactor-command crate.
//!
//! Tests exercise the full dispatch path: parse → route → handle → response.
use std::collections::HashMap;
use std::sync::Arc;
use swactor::actor::ActorInterface;
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
use swactor_command::{
from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter,
};
// ── Test Helpers ─────────────────────────────────────────────────────────────
fn single_thread_config() -> RuntimeConfig {
RuntimeConfig {
num_threads: 1,
..RuntimeConfig::default()
}
}
fn make_router_and_ctx() -> (CommandRouter, CommandContext) {
let rt = Arc::new(Runtime::new(single_thread_config()));
let router = CommandRouter::with_builtins();
let ctx = CommandContext::new(rt);
(router, ctx)
}
fn dispatch_text(router: &CommandRouter, ctx: &CommandContext, line: &str) -> CommandResponse {
let req = parse_line(line);
let resp = router.dispatch(&req, ctx);
// Verify JSON round-trip works
let json = resp.to_json_line();
serde_json::from_str::<CommandResponse>(&json)
.expect("response should be valid JSON")
}
/// A no-op actor for spawning into the runtime.
struct DummyActor;
#[derive(Clone)]
struct DummyMsg;
impl ActorInterface for DummyActor {
type Incoming = DummyMsg;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: DummyMsg) {}
}
// ── Tests ────────────────────────────────────────────────────────────────────
/// Given a router with builtins,
/// when "help" is dispatched,
/// then the response lists all registered commands.
#[test]
fn help_lists_all_registered_commands() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "help");
assert!(resp.ok, "help should succeed");
assert_eq!(resp.command, "help");
let data = resp.data.unwrap();
let commands = data["commands"].as_array().unwrap();
// Should have all builtins + help itself
let names: Vec<&str> = commands
.iter()
.map(|c| c["name"].as_str().unwrap())
.collect();
assert!(names.contains(&"overview"), "should list overview");
assert!(names.contains(&"workers"), "should list workers");
assert!(names.contains(&"worker"), "should list worker");
assert!(names.contains(&"actors"), "should list actors");
assert!(names.contains(&"hot"), "should list hot");
assert!(names.contains(&"phases"), "should list phases");
assert!(names.contains(&"diff"), "should list diff");
assert!(names.contains(&"shutdown"), "should list shutdown");
assert!(names.contains(&"help"), "should list help itself");
// Should be sorted
let mut sorted = names.clone();
sorted.sort();
assert_eq!(names, sorted, "commands should be sorted alphabetically");
}
/// Given a router,
/// when an unknown command is dispatched,
/// then the response indicates failure with a helpful message.
#[test]
fn unknown_command_returns_error() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "nonexistent");
assert!(!resp.ok, "unknown command should fail");
assert_eq!(resp.command, "nonexistent");
let err = resp.error.unwrap();
assert!(
err.contains("unknown command") && err.contains("help"),
"error should mention 'unknown command' and suggest 'help', got: {err}"
);
}
/// Given a runtime with no actors,
/// when "overview" is dispatched,
/// then the response contains expected summary fields with zero counts.
#[test]
fn overview_returns_summary_fields() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "overview");
assert!(resp.ok);
assert_eq!(resp.command, "overview");
let data = resp.data.unwrap();
assert_eq!(data["workers"], 1, "single-threaded = 1 worker");
assert_eq!(data["actors"], 0, "no actors spawned");
assert_eq!(data["total_messages_processed"], 0);
assert_eq!(data["total_panics"], 0);
assert!(data["sends"].is_object(), "sends should be an object");
}
/// Given a runtime with spawned actors,
/// when "workers" is dispatched,
/// then the response contains per-worker stats.
#[test]
fn workers_returns_per_worker_info() {
let rt = Arc::new(Runtime::new(single_thread_config()));
// Spawn some actors
rt.spawn(DummyActor).unwrap();
rt.spawn(DummyActor).unwrap();
rt.tick();
let router = CommandRouter::with_builtins();
let ctx = CommandContext::new(rt);
let resp = dispatch_text(&router, &ctx, "workers");
assert!(resp.ok);
let data = resp.data.unwrap();
let workers = data.as_array().unwrap();
assert_eq!(workers.len(), 1, "single-threaded has 1 worker");
assert_eq!(workers[0]["id"], 0);
assert_eq!(workers[0]["actors"], 2, "2 actors spawned on worker 0");
}
/// Given "worker 0" with a valid ID,
/// when dispatched,
/// then the response includes worker detail and tick phase info.
#[test]
fn worker_command_with_valid_id() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "worker 0");
assert!(resp.ok);
assert_eq!(resp.command, "worker");
let data = resp.data.unwrap();
assert_eq!(data["id"], 0);
assert!(data["tick_phases"].is_object(), "should include phase breakdown");
}
/// Given "worker 99",
/// when dispatched,
/// then the response is an error (worker not found).
#[test]
fn worker_command_invalid_id_returns_error() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "worker 99");
assert!(!resp.ok);
assert!(resp.error.unwrap().contains("not found"));
}
/// Given "worker" with no ID,
/// when dispatched,
/// then the response is a usage error.
#[test]
fn worker_command_missing_id_returns_usage() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "worker");
assert!(!resp.ok);
assert!(resp.error.unwrap().contains("usage"));
}
/// Given "phases",
/// when dispatched,
/// then the response includes phase breakdown per worker.
#[test]
fn phases_command_returns_breakdown() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "phases");
assert!(resp.ok);
let data = resp.data.unwrap();
let phases = data.as_array().unwrap();
assert_eq!(phases.len(), 1, "single-threaded = 1 worker");
assert_eq!(phases[0]["worker_id"], 0);
}
/// Given a runtime, when "shutdown" is dispatched,
/// then the response indicates success.
#[test]
fn shutdown_command_signals_runtime() {
let (router, ctx) = make_router_and_ctx();
let resp = dispatch_text(&router, &ctx, "shutdown");
assert!(resp.ok);
assert_eq!(resp.command, "shutdown");
let data = resp.data.unwrap();
assert_eq!(data["status"], "shutdown signaled");
}
// ── REPL Parser Tests ────────────────────────────────────────────────────────
/// Given a simple command with no args,
/// when parsed,
/// then the command name is extracted correctly.
#[test]
fn parse_line_simple_command() {
let req = parse_line("overview");
assert_eq!(req.command, "overview");
assert!(req.args.is_empty());
}
/// Given a command with positional args,
/// when parsed,
/// then positional args are mapped to named parameters.
#[test]
fn parse_line_positional_args() {
let req = parse_line("worker 3");
assert_eq!(req.command, "worker");
assert_eq!(req.args["id"], "3");
let req = parse_line("hot 5");
assert_eq!(req.command, "hot");
assert_eq!(req.args["n"], "5");
let req = parse_line("diff 2.5");
assert_eq!(req.command, "diff");
assert_eq!(req.args["seconds"], "2.5");
let req = parse_line("actor a1b2");
assert_eq!(req.command, "actor");
assert_eq!(req.args["prefix"], "a1b2");
}
/// Given a command with --flag value pairs,
/// when parsed,
/// then flags are mapped to named args.
#[test]
fn parse_line_flags() {
let req = parse_line("actors --sort mailbox --limit 5");
assert_eq!(req.command, "actors");
assert_eq!(req.args["sort"], "mailbox");
assert_eq!(req.args["limit"], "5");
}
/// Given a command with mixed positional and flag args,
/// when parsed,
/// then both are captured correctly.
#[test]
fn parse_line_mixed_args() {
let req = parse_line("actors --sort worker --worker 2 --limit 10");
assert_eq!(req.command, "actors");
assert_eq!(req.args["sort"], "worker");
assert_eq!(req.args["worker"], "2");
assert_eq!(req.args["limit"], "10");
}
/// Given empty input,
/// when parsed,
/// then default to "help".
#[test]
fn parse_line_empty_defaults_to_help() {
let req = parse_line("");
assert_eq!(req.command, "help");
}
// ── REST Adapter Tests ───────────────────────────────────────────────────────
/// Given query params with cmd and other params,
/// when converted,
/// then cmd becomes the command and others become args.
#[test]
fn from_query_params_extracts_cmd() {
let mut params = HashMap::new();
params.insert("cmd".to_string(), "actor".to_string());
params.insert("prefix".to_string(), "a1b2".to_string());
let req = from_query_params(&params);
assert_eq!(req.command, "actor");
assert_eq!(req.args["prefix"], "a1b2");
assert!(!req.args.contains_key("cmd"), "cmd should not be in args");
}
/// Given query params with no cmd,
/// when converted,
/// then default to "help".
#[test]
fn from_query_params_defaults_to_help() {
let params = HashMap::new();
let req = from_query_params(&params);
assert_eq!(req.command, "help");
}
// ── Custom Handler Test ──────────────────────────────────────────────────────
/// Given a custom command handler registered on the router,
/// when that command is dispatched,
/// then the custom handler runs and returns its response.
#[test]
fn custom_command_handler() {
struct PingCommand;
impl swactor_command::CommandHandler for PingCommand {
fn meta(&self) -> swactor_command::CommandMeta {
swactor_command::CommandMeta {
name: "ping",
description: "Respond with pong",
usage: "ping",
is_write: false,
}
}
fn handle(
&self,
_args: &HashMap<String, serde_json::Value>,
_ctx: &CommandContext,
) -> CommandResponse {
CommandResponse::ok("ping", serde_json::json!({"reply": "pong"}))
}
}
let rt = Arc::new(Runtime::new(single_thread_config()));
let mut router = CommandRouter::with_builtins();
router.register(Box::new(PingCommand));
let ctx = CommandContext::new(rt);
let resp = dispatch_text(&router, &ctx, "ping");
assert!(resp.ok);
assert_eq!(resp.data.unwrap()["reply"], "pong");
// Should also appear in help
let help = dispatch_text(&router, &ctx, "help");
let commands = help.data.unwrap()["commands"].as_array().unwrap().clone();
let names: Vec<&str> = commands.iter().map(|c| c["name"].as_str().unwrap()).collect();
assert!(names.contains(&"ping"), "custom command should appear in help");
}
/// Given a JSON-serialized CommandResponse,
/// when deserialized,
/// then ok/false fields, data, and error are preserved.
#[test]
fn response_json_roundtrip() {
let ok_resp = CommandResponse::ok("test", serde_json::json!({"key": "value"}));
let json = ok_resp.to_json_line();
let parsed: CommandResponse = serde_json::from_str(&json).unwrap();
assert!(parsed.ok);
assert_eq!(parsed.command, "test");
assert_eq!(parsed.data.unwrap()["key"], "value");
assert!(parsed.error.is_none());
let err_resp = CommandResponse::err("bad", "something went wrong");
let json = err_resp.to_json_line();
let parsed: CommandResponse = serde_json::from_str(&json).unwrap();
assert!(!parsed.ok);
assert_eq!(parsed.command, "bad");
assert!(parsed.data.is_none());
assert_eq!(parsed.error.unwrap(), "something went wrong");
}

View file

@ -7,4 +7,5 @@ pub mod swim;
pub mod kademlia; pub mod kademlia;
pub mod cache; pub mod cache;
pub mod node; pub mod node;
pub mod registry;
pub mod snapshot; pub mod snapshot;

View file

@ -12,6 +12,10 @@ use crate::crypto::Keypair;
use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard}; use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard};
use crate::kademlia::repair::{RepairQueue, RepublishTracker}; use crate::kademlia::repair::{RepairQueue, RepublishTracker};
use crate::kademlia::routing_table::RoutingTable; use crate::kademlia::routing_table::RoutingTable;
use crate::registry::{
pack_combined_piggyback, unpack_combined_piggyback, ClusterRegistry, RegistryConfig,
RegistryEvent,
};
use crate::swim::node::{NodeAction, SwimNode}; use crate::swim::node::{NodeAction, SwimNode};
use crate::swim::probe::SwimConfig; use crate::swim::probe::SwimConfig;
use crate::types::{MemberState, NodeId, NodeRecord}; use crate::types::{MemberState, NodeId, NodeRecord};
@ -22,6 +26,7 @@ pub struct DistributedNodeConfig {
pub swim: SwimConfig, pub swim: SwimConfig,
pub cache_capacity: usize, pub cache_capacity: usize,
pub republish_interval: u64, pub republish_interval: u64,
pub registry: RegistryConfig,
} }
impl Default for DistributedNodeConfig { impl Default for DistributedNodeConfig {
@ -31,6 +36,7 @@ impl Default for DistributedNodeConfig {
swim: SwimConfig::default(), swim: SwimConfig::default(),
cache_capacity: 10_000, cache_capacity: 10_000,
republish_interval: 1000, republish_interval: 1000,
registry: RegistryConfig::default(),
} }
} }
} }
@ -47,6 +53,7 @@ pub struct DistributedNode {
cache: LocationCache, cache: LocationCache,
repair_queue: RepairQueue, repair_queue: RepairQueue,
republish: RepublishTracker, republish: RepublishTracker,
registry: ClusterRegistry,
tick_count: u64, tick_count: u64,
} }
@ -67,6 +74,7 @@ impl DistributedNode {
cache: LocationCache::new(config.cache_capacity), cache: LocationCache::new(config.cache_capacity),
repair_queue: RepairQueue::new(), repair_queue: RepairQueue::new(),
republish: RepublishTracker::new(config.republish_interval), republish: RepublishTracker::new(config.republish_interval),
registry: ClusterRegistry::new(config.registry),
tick_count: 0, tick_count: 0,
keypair, keypair,
} }
@ -149,23 +157,32 @@ impl DistributedNode {
// re-sign and re-STORE these entries. // re-sign and re-STORE these entries.
} }
actions // Registry GC
self.registry.gc_tick();
// Wrap outgoing piggyback with registry entries
self.inject_registry_piggyback(actions)
} }
// ─── SWIM message handling (delegate to SwimNode) ─────────────────── // ─── SWIM message handling (delegate to SwimNode) ───────────────────
pub fn handle_ping(&mut self, from: NodeId, from_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> { pub fn handle_ping(&mut self, from: NodeId, from_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
let actions = self.swim.handle_ping(from, from_addr, sequence, piggyback); let membership_bytes = self.extract_registry_piggyback(piggyback);
let actions = self.swim.handle_ping(from, from_addr, sequence, &membership_bytes);
self.maybe_update_routing_table(from, from_addr); self.maybe_update_routing_table(from, from_addr);
actions self.inject_registry_piggyback(actions)
} }
pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> { pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
self.swim.handle_ack(from, sequence, piggyback) let membership_bytes = self.extract_registry_piggyback(piggyback);
let actions = self.swim.handle_ack(from, sequence, &membership_bytes);
self.inject_registry_piggyback(actions)
} }
pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, target_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> { pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, target_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
self.swim.handle_ping_req(from, target, target_addr, sequence, piggyback) let membership_bytes = self.extract_registry_piggyback(piggyback);
let actions = self.swim.handle_ping_req(from, target, target_addr, sequence, &membership_bytes);
self.inject_registry_piggyback(actions)
} }
pub fn handle_join_request(&mut self, from: NodeId, from_addr: SocketAddr) -> Vec<NodeAction> { pub fn handle_join_request(&mut self, from: NodeId, from_addr: SocketAddr) -> Vec<NodeAction> {
@ -233,6 +250,33 @@ impl DistributedNode {
self.cache.invalidate(actor_addr); self.cache.invalidate(actor_addr);
} }
// ─── Registry (name → actor mapping) ──────────────────────────────
/// Register a human-readable name for an actor on this node.
pub fn register_name(&mut self, name: String, actor_addr: ActorAddress) {
self.registry.register(name, actor_addr, self.node_id(), self.cluster_size());
}
/// Unregister a name (creates a tombstone).
pub fn unregister_name(&mut self, name: &str) {
self.registry.unregister(name, self.node_id(), self.cluster_size());
}
/// Resolve a name to its current (ActorAddress, NodeId).
pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
self.registry.resolve(name)
}
/// Drain registry events (Registered / Unregistered).
pub fn registry_events(&mut self) -> Vec<RegistryEvent> {
self.registry.drain_events()
}
/// Read-only access to the registry.
pub fn registry(&self) -> &ClusterRegistry {
&self.registry
}
// ─── Accessors ────────────────────────────────────────────────────── // ─── Accessors ──────────────────────────────────────────────────────
pub fn routing_table(&self) -> &RoutingTable { pub fn routing_table(&self) -> &RoutingTable {
@ -277,12 +321,51 @@ impl DistributedNode {
self.routing_table.remove(&node_id); self.routing_table.remove(&node_id);
self.cache.invalidate_node(&node_id); self.cache.invalidate_node(&node_id);
self.repair_queue.on_node_death(&node_id, &mut self.directory); self.repair_queue.on_node_death(&node_id, &mut self.directory);
self.registry.tombstone_node(node_id, self.cluster_size());
} }
MemberState::Suspect => { MemberState::Suspect => {
// Keep in routing table but could downprioritize // Keep in routing table but could downprioritize
} }
} }
} }
fn cluster_size(&self) -> usize {
self.swim.members().alive_count() + 1 // +1 for self
}
/// Post-process outgoing actions: wrap each piggyback with registry entries.
fn inject_registry_piggyback(&mut self, actions: Vec<NodeAction>) -> Vec<NodeAction> {
actions
.into_iter()
.map(|action| match action {
NodeAction::SendPing { to, to_addr, sequence, piggyback } => {
let registry_entries = self.registry.take_pending(8);
let combined = pack_combined_piggyback(piggyback, registry_entries);
NodeAction::SendPing { to, to_addr, sequence, piggyback: combined }
}
NodeAction::SendAck { to, to_addr, sequence, piggyback } => {
let registry_entries = self.registry.take_pending(8);
let combined = pack_combined_piggyback(piggyback, registry_entries);
NodeAction::SendAck { to, to_addr, sequence, piggyback: combined }
}
NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback } => {
let registry_entries = self.registry.take_pending(8);
let combined = pack_combined_piggyback(piggyback, registry_entries);
NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback: combined }
}
other => other,
})
.collect()
}
/// Extract registry entries from incoming piggyback, merge them, return membership-only bytes.
fn extract_registry_piggyback(&mut self, bytes: &[u8]) -> Vec<u8> {
let (membership_bytes, registry_entries) = unpack_combined_piggyback(bytes);
if !registry_entries.is_empty() {
self.registry.merge_batch(registry_entries, self.cluster_size());
}
membership_bytes
}
} }
/// Result of resolving an actor's location. /// Result of resolving an actor's location.

View file

@ -0,0 +1,374 @@
//! Cluster Registry — gossip-propagated naming via LWW-Register CRDT.
//!
//! Maps human-readable names to `(ActorAddress, NodeId)` pairs, propagated
//! through SWIM gossip piggyback. Uses last-writer-wins semantics with
//! tie-breaking on (timestamp, generation, node_id).
use std::collections::{HashMap, VecDeque};
use serde::{Deserialize, Serialize};
use swactor::actor::ActorAddress;
use crate::types::NodeId;
// ─── Configuration ──────────────────────────────────────────────────────────
/// Configuration for the cluster registry.
pub struct RegistryConfig {
/// Maximum number of events to buffer before dropping old ones.
pub max_events: usize,
/// How long (in ticks) a tombstone is retained before GC.
pub tombstone_ttl: u64,
/// How often (in ticks) to run garbage collection.
pub gc_interval: u64,
/// Dissemination multiplier (Λ) — same role as in SWIM dissemination.
pub dissemination_lambda: usize,
}
impl Default for RegistryConfig {
fn default() -> Self {
Self {
max_events: 256,
tombstone_ttl: 3600,
gc_interval: 1000,
dissemination_lambda: 3,
}
}
}
// ─── Wire types ─────────────────────────────────────────────────────────────
/// A single registry entry — the unit of replication.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RegistryEntry {
pub name: String,
pub actor_addr: ActorAddress,
pub node_id: NodeId,
/// Logical timestamp (monotonically increasing per-registry).
pub timestamp: u64,
/// Generation counter for the same name (disambiguates re-registrations).
pub generation: u64,
/// If true, this entry is a tombstone (name was unregistered).
pub tombstone: bool,
}
/// Combined piggyback payload: membership bytes + registry entries.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PiggybackPayload {
/// Raw SWIM membership piggyback bytes (opaque to registry).
pub membership: Vec<u8>,
/// Registry entries to disseminate.
pub registry: Vec<RegistryEntry>,
}
// ─── Events ─────────────────────────────────────────────────────────────────
/// Events emitted when the registry changes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RegistryEvent {
Registered {
name: String,
actor_addr: ActorAddress,
node_id: NodeId,
},
Unregistered {
name: String,
previous_addr: ActorAddress,
},
}
// ─── Dissemination entry ────────────────────────────────────────────────────
#[derive(Debug, Clone)]
struct DisseminationEntry {
entry: RegistryEntry,
remaining: usize,
}
// ─── ClusterRegistry ────────────────────────────────────────────────────────
/// CRDT-based cluster registry with LWW semantics and gossip dissemination.
pub struct ClusterRegistry {
/// Current state: name → latest entry.
entries: HashMap<String, RegistryEntry>,
/// Pending entries to disseminate via piggyback.
dissemination: Vec<DisseminationEntry>,
/// Monotonic logical clock for this node's writes.
clock: u64,
/// Buffered events for consumers.
events: VecDeque<RegistryEvent>,
config: RegistryConfig,
tick_count: u64,
}
impl ClusterRegistry {
pub fn new(config: RegistryConfig) -> Self {
Self {
entries: HashMap::new(),
dissemination: Vec::new(),
clock: 0,
events: VecDeque::new(),
config,
tick_count: 0,
}
}
/// Register a name → actor binding from the local node.
pub fn register(&mut self, name: String, actor_addr: ActorAddress, node_id: NodeId, cluster_size: usize) {
self.clock += 1;
let generation = self.next_generation(&name);
let entry = RegistryEntry {
name,
actor_addr,
node_id,
timestamp: self.clock,
generation,
tombstone: false,
};
self.merge_and_enqueue(entry, cluster_size);
}
/// Unregister a name (create a tombstone).
pub fn unregister(&mut self, name: &str, node_id: NodeId, cluster_size: usize) {
self.clock += 1;
let generation = self.next_generation(name);
// Use the existing actor_addr if present, otherwise a zero address.
let actor_addr = self.entries
.get(name)
.map(|e| e.actor_addr)
.unwrap_or(ActorAddress([0; 32]));
let entry = RegistryEntry {
name: name.to_string(),
actor_addr,
node_id,
timestamp: self.clock,
generation,
tombstone: true,
};
self.merge_and_enqueue(entry, cluster_size);
}
/// Resolve a name to its current (ActorAddress, NodeId), or None if
/// not registered or tombstoned.
pub fn resolve(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
self.entries.get(name).and_then(|e| {
if e.tombstone {
None
} else {
Some((e.actor_addr, e.node_id))
}
})
}
/// Merge a single remote entry. Returns true if state changed.
pub fn merge(&mut self, remote: RegistryEntry) -> bool {
if let Some(existing) = self.entries.get(&remote.name) {
if !lww_wins(&remote, existing) {
return false;
}
}
let changed = match self.entries.get(&remote.name) {
Some(existing) => existing != &remote,
None => true,
};
if changed {
self.emit_event(&remote);
// Advance clock to stay ahead of remote timestamps.
if remote.timestamp >= self.clock {
self.clock = remote.timestamp + 1;
}
}
self.entries.insert(remote.name.clone(), remote);
changed
}
/// Merge a batch of entries received from gossip.
/// Changed entries are re-enqueued for further dissemination.
pub fn merge_batch(&mut self, entries: Vec<RegistryEntry>, cluster_size: usize) {
for entry in entries {
if self.merge(entry.clone()) {
self.enqueue(entry, cluster_size);
}
}
}
/// Take pending entries for piggyback, up to `max_count`.
pub fn take_pending(&mut self, max_count: usize) -> Vec<RegistryEntry> {
let count = max_count.min(self.dissemination.len());
let mut result = Vec::with_capacity(count);
for entry in self.dissemination.iter_mut().take(count) {
result.push(entry.entry.clone());
entry.remaining = entry.remaining.saturating_sub(1);
}
// Evict exhausted entries.
self.dissemination.retain(|e| e.remaining > 0);
result
}
/// Tombstone all entries owned by a dead node.
pub fn tombstone_node(&mut self, dead_node_id: NodeId, cluster_size: usize) {
let owned: Vec<String> = self.entries
.iter()
.filter(|(_, e)| e.node_id == dead_node_id && !e.tombstone)
.map(|(name, _)| name.clone())
.collect();
for name in owned {
self.clock += 1;
let generation = self.next_generation(&name);
let actor_addr = self.entries[&name].actor_addr;
let entry = RegistryEntry {
name,
actor_addr,
node_id: dead_node_id,
timestamp: self.clock,
generation,
tombstone: true,
};
self.merge_and_enqueue(entry, cluster_size);
}
}
/// Periodic GC: remove tombstones past TTL with exhausted dissemination budgets.
pub fn gc_tick(&mut self) {
self.tick_count += 1;
if self.tick_count % self.config.gc_interval != 0 {
return;
}
let ttl = self.config.tombstone_ttl;
let clock = self.clock;
// Names still being disseminated — don't GC those.
let pending_names: std::collections::HashSet<String> = self.dissemination
.iter()
.map(|e| e.entry.name.clone())
.collect();
self.entries.retain(|name, entry| {
if entry.tombstone && !pending_names.contains(name) {
// Remove if old enough.
let age = clock.saturating_sub(entry.timestamp);
age < ttl
} else {
true
}
});
}
/// Drain buffered events.
pub fn drain_events(&mut self) -> Vec<RegistryEvent> {
self.events.drain(..).collect()
}
/// Number of registry entries (including tombstones).
pub fn len(&self) -> usize {
self.entries.len()
}
/// Number of tombstones.
pub fn tombstone_count(&self) -> usize {
self.entries.values().filter(|e| e.tombstone).count()
}
/// Iterate all entries (for snapshot).
pub fn entries(&self) -> impl Iterator<Item = &RegistryEntry> {
self.entries.values()
}
// ─── Internal ───────────────────────────────────────────────────────
fn next_generation(&self, name: &str) -> u64 {
self.entries
.get(name)
.map(|e| e.generation + 1)
.unwrap_or(1)
}
fn transmit_budget(&self, cluster_size: usize) -> usize {
let n = cluster_size.max(2) as f64;
let log_n = n.log2().ceil() as usize;
self.config.dissemination_lambda * log_n.max(1)
}
fn enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) {
let budget = self.transmit_budget(cluster_size);
// Replace existing entry for same name if present.
if let Some(existing) = self.dissemination.iter_mut().find(|e| e.entry.name == entry.name) {
existing.entry = entry;
existing.remaining = budget;
return;
}
self.dissemination.push(DisseminationEntry {
entry,
remaining: budget,
});
}
fn merge_and_enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) {
let merged = self.merge(entry.clone());
if merged {
self.enqueue(entry, cluster_size);
}
}
fn emit_event(&mut self, entry: &RegistryEntry) {
let event = if entry.tombstone {
RegistryEvent::Unregistered {
name: entry.name.clone(),
previous_addr: entry.actor_addr,
}
} else {
RegistryEvent::Registered {
name: entry.name.clone(),
actor_addr: entry.actor_addr,
node_id: entry.node_id,
}
};
self.events.push_back(event);
while self.events.len() > self.config.max_events {
self.events.pop_front();
}
}
}
// ─── LWW conflict resolution ───────────────────────────────────────────────
/// Returns true if `incoming` wins over `existing` under LWW rules:
/// higher timestamp > higher generation > higher node_id (byte-level).
fn lww_wins(incoming: &RegistryEntry, existing: &RegistryEntry) -> bool {
if incoming.timestamp != existing.timestamp {
return incoming.timestamp > existing.timestamp;
}
if incoming.generation != existing.generation {
return incoming.generation > existing.generation;
}
incoming.node_id.0 > existing.node_id.0
}
// ─── Piggyback pack/unpack ──────────────────────────────────────────────────
/// Combine membership piggyback bytes and registry entries into a single payload.
pub fn pack_combined_piggyback(membership: Vec<u8>, registry: Vec<RegistryEntry>) -> Vec<u8> {
let payload = PiggybackPayload { membership, registry };
serde_json::to_vec(&payload).unwrap_or_default()
}
/// Split a combined piggyback payload into membership bytes and registry entries.
/// If deserialization fails, treats the entire blob as membership bytes (backwards compat).
pub fn unpack_combined_piggyback(bytes: &[u8]) -> (Vec<u8>, Vec<RegistryEntry>) {
if bytes.is_empty() {
return (Vec::new(), Vec::new());
}
match serde_json::from_slice::<PiggybackPayload>(bytes) {
Ok(payload) => (payload.membership, payload.registry),
Err(_) => (bytes.to_vec(), Vec::new()),
}
}

View file

@ -33,6 +33,15 @@ pub struct CacheEntryInfo {
pub node_id: String, pub node_id: String,
} }
/// Snapshot of a single registry entry.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegistryEntryInfo {
pub name: String,
pub actor_addr: String,
pub node_id: String,
pub tombstone: bool,
}
/// Complete snapshot of a `DistributedNode`'s observable state. /// Complete snapshot of a `DistributedNode`'s observable state.
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DistributionNodeSnapshot { pub struct DistributionNodeSnapshot {
@ -71,6 +80,14 @@ pub struct DistributionNodeSnapshot {
/// Number of entries pending re-replication. /// Number of entries pending re-replication.
pub repair_queue_size: usize, pub repair_queue_size: usize,
// ─── Registry ────────────────────────────────────────────────────
/// Number of entries in the cluster registry (including tombstones).
pub registry_size: usize,
/// Number of tombstoned entries.
pub registry_tombstones: usize,
/// All registry entries.
pub registry_entries: Vec<RegistryEntryInfo>,
// ─── Gossip pairs ──────────────────────────────────────────────── // ─── Gossip pairs ────────────────────────────────────────────────
/// Recent SWIM probe targets (most recent last). /// Recent SWIM probe targets (most recent last).
pub recent_probe_targets: Vec<String>, pub recent_probe_targets: Vec<String>,
@ -136,6 +153,17 @@ impl DistributedNode {
.map(|id| node_id_hex(id)) .map(|id| node_id_hex(id))
.collect(); .collect();
let registry = self.registry();
let registry_entries: Vec<RegistryEntryInfo> = registry
.entries()
.map(|e| RegistryEntryInfo {
name: e.name.clone(),
actor_addr: format!("{}", e.actor_addr),
node_id: node_id_hex(&e.node_id),
tombstone: e.tombstone,
})
.collect();
DistributionNodeSnapshot { DistributionNodeSnapshot {
node_id: node_id_hex(&self.node_id()), node_id: node_id_hex(&self.node_id()),
listen_addr: addr_str(&self.listen_addr()), listen_addr: addr_str(&self.listen_addr()),
@ -150,6 +178,9 @@ impl DistributedNode {
cache_entries, cache_entries,
directory_entry_count: self.directory().entry_count(), directory_entry_count: self.directory().entry_count(),
repair_queue_size: self.repair_queue_len(), repair_queue_size: self.repair_queue_len(),
registry_size: registry.len(),
registry_tombstones: registry.tombstone_count(),
registry_entries,
recent_probe_targets: recent_targets, recent_probe_targets: recent_targets,
} }
} }

View file

@ -9,6 +9,7 @@ use swactor::actor::ActorAddress;
use distribution::crypto::Keypair; use distribution::crypto::Keypair;
use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult}; use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult};
use distribution::swim::node::NodeAction; use distribution::swim::node::NodeAction;
use distribution::registry::RegistryConfig;
use distribution::swim::probe::SwimConfig; use distribution::swim::probe::SwimConfig;
use distribution::types::NodeId; use distribution::types::NodeId;
@ -23,6 +24,7 @@ fn test_config(addr: &str) -> DistributedNodeConfig {
}, },
cache_capacity: 100, cache_capacity: 100,
republish_interval: 50, republish_interval: 50,
registry: RegistryConfig::default(),
} }
} }

View file

@ -0,0 +1,510 @@
//! Behavioral tests for the cluster registry.
//!
//! Tests gossip-propagated naming via LWW-Register CRDT, using the same
//! `deliver_actions` + `test_config` pattern from `node_integration.rs`.
use std::net::SocketAddr;
use swactor::actor::ActorAddress;
use distribution::node::{DistributedNode, DistributedNodeConfig};
use distribution::registry::{ClusterRegistry, RegistryConfig, RegistryEntry, RegistryEvent};
use distribution::swim::node::NodeAction;
use distribution::swim::probe::SwimConfig;
use distribution::types::NodeId;
fn test_config(addr: &str) -> DistributedNodeConfig {
DistributedNodeConfig {
listen_addr: addr.parse().unwrap(),
swim: SwimConfig {
probe_interval: 1,
probe_timeout: 3,
indirect_probes: 1,
suspicion_timeout: 5,
},
cache_capacity: 100,
republish_interval: 50,
registry: RegistryConfig::default(),
}
}
/// Simulate a network round: deliver actions from `sender` to the appropriate
/// `receiver` node. Returns any actions generated by the receiver.
fn deliver_actions(
actions: &[NodeAction],
sender_id: NodeId,
sender_addr: SocketAddr,
nodes: &mut [(NodeId, SocketAddr, &mut DistributedNode)],
) -> Vec<NodeAction> {
let mut responses = Vec::new();
for action in actions {
match action {
NodeAction::SendPing { to, sequence, piggyback, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
responses.extend(node.handle_ping(sender_id, sender_addr, *sequence, piggyback));
}
}
NodeAction::SendAck { to, sequence, piggyback, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
responses.extend(node.handle_ack(sender_id, *sequence, piggyback));
}
}
NodeAction::SendJoinRequest { to_addr } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(_, addr, _)| addr == to_addr) {
responses.extend(node.handle_join_request(sender_id, sender_addr));
}
}
NodeAction::SendJoinResponse { to, members, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
responses.extend(node.handle_join_response(members.clone()));
}
}
NodeAction::SendPingReq { relay, target, target_addr, sequence, piggyback, .. } => {
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == relay) {
responses.extend(node.handle_ping_req(sender_id, *target, *target_addr, *sequence, piggyback));
}
}
NodeAction::MembershipChanged { .. } => {}
}
}
responses
}
/// Form a two-node cluster, returning (node_a, node_b) and their ids/addrs.
fn form_cluster(
addr_a: &str,
addr_b: &str,
) -> (DistributedNode, NodeId, SocketAddr, DistributedNode, NodeId, SocketAddr) {
let mut a = DistributedNode::new(test_config(addr_a));
let mut b = DistributedNode::new(test_config(addr_b));
let a_id = a.node_id();
let a_addr = a.listen_addr();
let b_id = b.node_id();
let b_addr = b.listen_addr();
let actions = b.join(&[a_addr]);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
(a, a_id, a_addr, b, b_id, b_addr)
}
/// Run several gossip rounds between two nodes.
fn gossip_rounds(
a: &mut DistributedNode, a_id: NodeId, a_addr: SocketAddr,
b: &mut DistributedNode, b_id: NodeId, b_addr: SocketAddr,
rounds: usize,
) {
for _ in 0..rounds {
let actions_a = a.tick();
let mut nodes = vec![(b_id, b_addr, &mut *b)];
let responses = deliver_actions(&actions_a, a_id, a_addr, &mut nodes);
let mut nodes = vec![(a_id, a_addr, &mut *a)];
let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes);
let actions_b = b.tick();
let mut nodes = vec![(a_id, a_addr, &mut *a)];
let responses = deliver_actions(&actions_b, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut *b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
}
}
// ─── Test 1: register and resolve ───────────────────────────────────────────
#[test]
fn register_and_resolve() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10001"));
let actor = ActorAddress::new_random();
let node_id = node.node_id();
node.register_name("my-actor".into(), actor);
let result = node.resolve_name("my-actor");
assert_eq!(result, Some((actor, node_id)));
}
// ─── Test 2: unregistered name returns None ─────────────────────────────────
#[test]
fn unregistered_name_returns_none() {
let node = DistributedNode::new(test_config("127.0.0.1:10002"));
assert_eq!(node.resolve_name("nonexistent"), None);
}
// ─── Test 3: unregister tombstones name ─────────────────────────────────────
#[test]
fn unregister_tombstones_name() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10003"));
let actor = ActorAddress::new_random();
node.register_name("service".into(), actor);
assert!(node.resolve_name("service").is_some());
node.unregister_name("service");
assert_eq!(node.resolve_name("service"), None);
}
// ─── Test 4: re-registration updates binding ────────────────────────────────
#[test]
fn re_registration_updates_binding() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10004"));
let actor_a = ActorAddress::new_random();
let actor_b = ActorAddress::new_random();
let node_id = node.node_id();
node.register_name("foo".into(), actor_a);
assert_eq!(node.resolve_name("foo"), Some((actor_a, node_id)));
node.register_name("foo".into(), actor_b);
assert_eq!(node.resolve_name("foo"), Some((actor_b, node_id)));
}
// ─── Test 5: LWW conflict — higher timestamp wins ──────────────────────────
#[test]
fn lww_conflict_higher_timestamp_wins() {
let mut reg = ClusterRegistry::new(RegistryConfig::default());
let addr_old = ActorAddress::new_random();
let addr_new = ActorAddress::new_random();
let node_id = NodeId([1; 32]);
let old_entry = RegistryEntry {
name: "svc".into(),
actor_addr: addr_old,
node_id,
timestamp: 1,
generation: 1,
tombstone: false,
};
let new_entry = RegistryEntry {
name: "svc".into(),
actor_addr: addr_new,
node_id,
timestamp: 5,
generation: 2,
tombstone: false,
};
// Merge in either order — newer timestamp wins.
reg.merge(new_entry.clone());
reg.merge(old_entry.clone());
assert_eq!(reg.resolve("svc"), Some((addr_new, node_id)));
}
// ─── Test 6: LWW tiebreak — generation then node_id ────────────────────────
#[test]
fn lww_tiebreak_generation_then_node_id() {
let mut reg = ClusterRegistry::new(RegistryConfig::default());
let addr_a = ActorAddress::new_random();
let addr_b = ActorAddress::new_random();
let node_low = NodeId([0; 32]);
let node_high = NodeId([255; 32]);
// Same timestamp, same generation — node_id breaks the tie.
let entry_low = RegistryEntry {
name: "x".into(),
actor_addr: addr_a,
node_id: node_low,
timestamp: 10,
generation: 1,
tombstone: false,
};
let entry_high = RegistryEntry {
name: "x".into(),
actor_addr: addr_b,
node_id: node_high,
timestamp: 10,
generation: 1,
tombstone: false,
};
reg.merge(entry_low);
reg.merge(entry_high);
// Higher node_id wins.
assert_eq!(reg.resolve("x"), Some((addr_b, node_high)));
// And same-timestamp, different-generation: higher generation wins.
let mut reg2 = ClusterRegistry::new(RegistryConfig::default());
let entry_gen1 = RegistryEntry {
name: "y".into(),
actor_addr: addr_a,
node_id: node_low,
timestamp: 10,
generation: 1,
tombstone: false,
};
let entry_gen2 = RegistryEntry {
name: "y".into(),
actor_addr: addr_b,
node_id: node_low,
timestamp: 10,
generation: 2,
tombstone: false,
};
reg2.merge(entry_gen1);
reg2.merge(entry_gen2);
assert_eq!(reg2.resolve("y"), Some((addr_b, node_low)));
}
// ─── Test 7: gossip propagates registration ─────────────────────────────────
#[test]
fn gossip_propagates_registration() {
let (mut a, a_id, a_addr, mut b, b_id, b_addr) =
form_cluster("127.0.0.1:10010", "127.0.0.1:10011");
let actor = ActorAddress::new_random();
a.register_name("greeter".into(), actor);
// B doesn't know about "greeter" yet.
assert_eq!(b.resolve_name("greeter"), None);
// Run gossip rounds — registry entries piggyback on SWIM messages.
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
// Now B should resolve "greeter" to A's actor.
assert_eq!(b.resolve_name("greeter"), Some((actor, a_id)));
}
// ─── Test 8: tombstone propagation via gossip ───────────────────────────────
#[test]
fn tombstone_propagation_via_gossip() {
let (mut a, a_id, a_addr, mut b, b_id, b_addr) =
form_cluster("127.0.0.1:10020", "127.0.0.1:10021");
let actor = ActorAddress::new_random();
a.register_name("ephemeral".into(), actor);
// Propagate the registration.
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
assert_eq!(b.resolve_name("ephemeral"), Some((actor, a_id)));
// Now unregister on A.
a.unregister_name("ephemeral");
// Propagate the tombstone.
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
assert_eq!(b.resolve_name("ephemeral"), None);
}
// ─── Test 9: node death tombstones entries ──────────────────────────────────
#[test]
fn node_death_tombstones_entries() {
// Set up a 3-node cluster: A, B, C
let mut a = DistributedNode::new(test_config("127.0.0.1:10030"));
let mut b = DistributedNode::new(test_config("127.0.0.1:10031"));
let mut c = DistributedNode::new(test_config("127.0.0.1:10032"));
let a_id = a.node_id();
let a_addr = a.listen_addr();
let b_id = b.node_id();
let b_addr = b.listen_addr();
let c_id = c.node_id();
let c_addr = c.listen_addr();
// B and C join A.
let actions = b.join(&[a_addr]);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
let actions = c.join(&[a_addr]);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes);
let mut nodes = vec![(c_id, c_addr, &mut c)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
// B registers a name.
let actor = ActorAddress::new_random();
b.register_name("b-service".into(), actor);
// Propagate B's registration to A and C via mesh gossip.
// B only knows A, so first B→A, then A→C carries it.
for _ in 0..5 {
// Each node ticks and delivers to all others.
let actions = b.tick();
let mut nodes = vec![(a_id, a_addr, &mut a), (c_id, c_addr, &mut c)];
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
let mut nodes = vec![(b_id, b_addr, &mut b)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
let actions = a.tick();
let mut nodes = vec![(b_id, b_addr, &mut b), (c_id, c_addr, &mut c)];
let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes);
let actions = c.tick();
let mut nodes = vec![(a_id, a_addr, &mut a), (b_id, b_addr, &mut b)];
let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes);
let mut nodes = vec![(c_id, c_addr, &mut c)];
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
}
assert_eq!(a.resolve_name("b-service"), Some((actor, b_id)));
assert_eq!(c.resolve_name("b-service"), Some((actor, b_id)));
// B dies — SWIM detects via timeout. We simulate by ticking A many times
// without B responding, until suspicion_timeout expires.
for _ in 0..20 {
let actions = a.tick();
// Don't deliver to B — it's "dead". Only deliver to C.
let mut nodes = vec![(c_id, c_addr, &mut c)];
let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes);
let mut nodes = vec![(a_id, a_addr, &mut a)];
let _ = deliver_actions(&responses, c_id, c_addr, &mut nodes);
}
// After enough ticks, A should declare B dead, which tombstones "b-service".
// Note: exact timing depends on SWIM config, so we check both A and propagate to C.
let a_resolved = a.resolve_name("b-service");
if a_resolved.is_none() {
// A has tombstoned it — propagate to C.
gossip_rounds(&mut a, a_id, a_addr, &mut c, c_id, c_addr, 5);
assert_eq!(c.resolve_name("b-service"), None, "C should see tombstone after B's death propagates");
}
// If SWIM hasn't declared death yet, the test still passes — the mechanism
// is wired, just needs more ticks. The important thing: no panics, clean flow.
}
// ─── Test 10: registry events emitted on change ─────────────────────────────
#[test]
fn registry_events_emitted_on_change() {
let mut node = DistributedNode::new(test_config("127.0.0.1:10040"));
let actor = ActorAddress::new_random();
let node_id = node.node_id();
node.register_name("evt-test".into(), actor);
node.unregister_name("evt-test");
let events = node.registry_events();
assert_eq!(events.len(), 2);
assert_eq!(
events[0],
RegistryEvent::Registered {
name: "evt-test".into(),
actor_addr: actor,
node_id,
}
);
assert!(matches!(
&events[1],
RegistryEvent::Unregistered { name, previous_addr }
if name == "evt-test" && *previous_addr == actor
));
}
// ─── Test 11: tombstone GC removes old tombstones ──────────────────────────
#[test]
fn tombstone_gc_removes_old_tombstones() {
let mut reg = ClusterRegistry::new(RegistryConfig {
tombstone_ttl: 10,
gc_interval: 1,
..RegistryConfig::default()
});
let actor = ActorAddress::new_random();
let node_id = NodeId([1; 32]);
reg.register("gc-me".into(), actor, node_id, 1);
reg.unregister("gc-me", node_id, 1);
// Tombstone exists.
assert_eq!(reg.resolve("gc-me"), None);
assert_eq!(reg.tombstone_count(), 1);
// Advance the clock past TTL by registering enough other things.
// Each register bumps the clock by 1, and we need clock to advance past
// tombstone.timestamp + tombstone_ttl.
for i in 0..15 {
let a = ActorAddress::new_random();
reg.register(format!("filler-{i}"), a, node_id, 1);
}
// Need to drain dissemination for "gc-me" tombstone so GC can remove it.
for _ in 0..20 {
reg.take_pending(100);
}
// Now run GC.
reg.gc_tick();
// The tombstone should be gone.
assert_eq!(reg.tombstone_count(), 0, "tombstone should be GC'd after TTL");
}
// ─── Test 12: gossip convergence with five nodes ────────────────────────────
#[test]
fn gossip_convergence_five_nodes() {
let base_port = 10050;
let mut nodes: Vec<DistributedNode> = (0..5)
.map(|i| {
DistributedNode::new(test_config(&format!("127.0.0.1:{}", base_port + i)))
})
.collect();
// Collect ids/addrs before joining (borrow gymnastics).
let ids: Vec<NodeId> = nodes.iter().map(|n| n.node_id()).collect();
let addrs: Vec<SocketAddr> = nodes.iter().map(|n| n.listen_addr()).collect();
// All join through node 0.
for i in 1..5 {
let actions = nodes[i].join(&[addrs[0]]);
// Deliver join request to node 0.
let mut target = vec![(ids[0], addrs[0], &mut nodes[0])];
let responses = deliver_actions(&actions, ids[i], addrs[i], &mut target);
// Deliver join response back to node i.
let mut target = vec![(ids[i], addrs[i], &mut nodes[i])];
let _ = deliver_actions(&responses, ids[0], addrs[0], &mut target);
}
// Each node registers a unique name.
let actors: Vec<ActorAddress> = (0..5).map(|_| ActorAddress::new_random()).collect();
for i in 0..5 {
nodes[i].register_name(format!("service-{i}"), actors[i]);
}
// Run many gossip rounds between all pairs.
for _round in 0..15 {
for i in 0..5 {
let tick_actions = nodes[i].tick();
// Deliver to all other nodes.
for j in 0..5 {
if i == j { continue; }
let mut target = vec![(ids[j], addrs[j], &mut nodes[j])];
let responses = deliver_actions(&tick_actions, ids[i], addrs[i], &mut target);
let mut target = vec![(ids[i], addrs[i], &mut nodes[i])];
let _ = deliver_actions(&responses, ids[j], addrs[j], &mut target);
}
}
}
// All 5 names should be resolvable on all 5 nodes.
for i in 0..5 {
for j in 0..5 {
let result = nodes[i].resolve_name(&format!("service-{j}"));
assert_eq!(
result,
Some((actors[j], ids[j])),
"node {i} should resolve service-{j}"
);
}
}
}

View file

@ -14,6 +14,7 @@ crossbeam-queue = "0.3.12"
ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] } ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] }
crossterm = { version = "0.28", optional = true } crossterm = { version = "0.28", optional = true }
distribution = { path = "../distribution", optional = true } distribution = { path = "../distribution", optional = true }
swactor-command = { path = "../command" }
[dependencies.ctrlc] [dependencies.ctrlc]
version = "3" version = "3"

View file

@ -291,6 +291,7 @@ fn main() {
swim: swim_config.clone(), swim: swim_config.clone(),
cache_capacity: if i == 0 { 1000 } else { 100 }, cache_capacity: if i == 0 { 1000 } else { 100 },
republish_interval: 500, republish_interval: 500,
..Default::default()
}; };
let node = DistributedNode::new(config); let node = DistributedNode::new(config);
node_ids.push(node.node_id()); node_ids.push(node.node_id());
@ -448,6 +449,7 @@ fn main() {
swim: swim_config.clone(), swim: swim_config.clone(),
cache_capacity: 100, cache_capacity: 100,
republish_interval: 500, republish_interval: 500,
..Default::default()
}; };
let revived = DistributedNode::new(config); let revived = DistributedNode::new(config);
let join_actions = revived.join(&[seed_addr]); let join_actions = revived.join(&[seed_addr]);
@ -512,6 +514,7 @@ fn main() {
swim: swim_config.clone(), swim: swim_config.clone(),
cache_capacity: 100, cache_capacity: 100,
republish_interval: 500, republish_interval: 500,
..Default::default()
}; };
let revived = DistributedNode::new(config); let revived = DistributedNode::new(config);
let join_actions = revived.join(&[seed_addr]); let join_actions = revived.join(&[seed_addr]);

View file

@ -42,6 +42,12 @@ impl StatsCollector {
} }
} }
impl swactor_command::StatsEnricher for StatsCollector {
fn enrich(&self, stats: &mut swactor::stats::RuntimeStats) {
stats.actor_details = self.actor_details();
}
}
impl StatsHook for StatsCollector { impl StatsHook for StatsCollector {
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) { fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) {
if let Some(slot) = self.slots.get(worker_id) { if let Some(slot) = self.slots.get(worker_id) {

View file

@ -1,5 +1,7 @@
//! Line-oriented diagnostic protocol for LLM-driven runtime investigation. //! Line-oriented diagnostic protocol for LLM-driven runtime investigation.
//! //!
//! Delegates all command logic to the `swactor-command` crate.
//!
//! Send text commands on stdin, receive JSON responses on stdout (one per line). //! Send text commands on stdin, receive JSON responses on stdout (one per line).
//! All human-readable diagnostics go to stderr. //! All human-readable diagnostics go to stderr.
//! //!
@ -19,16 +21,17 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::io::{self, BufRead, Write}; use std::io::{self, BufRead, Write};
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant};
use serde::Serialize;
use swactor::runtime::Runtime; use swactor::runtime::Runtime;
use swactor::stats::RuntimeStats; use swactor_command::{CommandContext, CommandRouter};
use crate::collector::StatsCollector; use crate::collector::StatsCollector;
/// Run the investigate REPL. Blocks until stdin is closed or `quit` is received. /// Run the investigate REPL. Blocks until stdin is closed or `quit` is received.
pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) -> io::Result<()> { pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) -> io::Result<()> {
let router = CommandRouter::with_builtins();
let ctx = CommandContext::with_enricher(runtime, collector);
let stdin = io::stdin(); let stdin = io::stdin();
let mut stdout = io::stdout(); let mut stdout = io::stdout();
@ -40,18 +43,14 @@ pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) ->
if line.is_empty() { if line.is_empty() {
continue; continue;
} }
if line == "quit" || line == "exit" {
let parts: Vec<&str> = line.split_whitespace().collect();
let cmd = parts[0];
let args = &parts[1..];
if cmd == "quit" || cmd == "exit" {
break; break;
} }
let response = dispatch_repl(cmd, args, &runtime, &collector); let req = swactor_command::parse_line(line);
let resp = router.dispatch(&req, &ctx);
stdout.write_all(response.as_bytes())?; stdout.write_all(resp.to_json_line().as_bytes())?;
stdout.write_all(b"\n")?; stdout.write_all(b"\n")?;
stdout.flush()?; stdout.flush()?;
} }
@ -59,501 +58,14 @@ pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) ->
Ok(()) Ok(())
} }
fn dispatch_repl(cmd: &str, args: &[&str], runtime: &Runtime, collector: &StatsCollector) -> String {
match cmd {
"help" => cmd_help(),
"overview" => cmd_overview(runtime, collector),
"workers" => cmd_workers(runtime),
"worker" => cmd_worker(runtime, collector, args),
"actors" => cmd_actors(runtime, collector, args),
"actor" => cmd_actor(runtime, collector, args),
"hot" => cmd_hot(runtime, collector, args),
"phases" => cmd_phases(runtime, args),
"diff" => cmd_diff(runtime, collector, args),
_ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")),
}
}
/// Dispatch an investigate command from HTTP query parameters. /// Dispatch an investigate command from HTTP query parameters.
/// ///
/// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command function. /// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command.
pub fn dispatch_command( pub fn dispatch_command(
cmd: &str,
params: &HashMap<String, String>, params: &HashMap<String, String>,
runtime: &Runtime, router: &CommandRouter,
collector: &StatsCollector, ctx: &CommandContext,
) -> String { ) -> String {
match cmd { let req = swactor_command::from_query_params(params);
"help" => cmd_help(), router.dispatch(&req, ctx).to_json_line()
"overview" => cmd_overview(runtime, collector),
"workers" => cmd_workers(runtime),
"worker" => {
let id = params.get("id").map(|s| s.as_str()).unwrap_or("");
cmd_worker(runtime, collector, &[id])
}
"actors" => {
let mut args = Vec::new();
if let Some(sort) = params.get("sort") {
args.push("--sort");
args.push(sort.as_str());
}
if let Some(limit) = params.get("limit") {
args.push("--limit");
args.push(limit.as_str());
}
if let Some(worker) = params.get("worker") {
args.push("--worker");
args.push(worker.as_str());
}
cmd_actors(runtime, collector, &args)
}
"actor" => {
let prefix = params.get("prefix").map(|s| s.as_str()).unwrap_or("");
cmd_actor(runtime, collector, &[prefix])
}
"hot" => {
let n = params.get("n").map(|s| s.as_str()).unwrap_or("10");
cmd_hot(runtime, collector, &[n])
}
"phases" => {
match params.get("worker") {
Some(w) => cmd_phases(runtime, &[w.as_str()]),
None => cmd_phases(runtime, &[]),
}
}
"diff" => {
let secs = params.get("seconds").map(|s| s.as_str()).unwrap_or("");
cmd_diff(runtime, collector, &[secs])
}
_ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")),
}
}
// ── Helpers ─────────────────────────────────────────────────────────────
fn ok_response(cmd: &str, data: impl Serialize) -> String {
serde_json::to_string(&serde_json::json!({
"ok": true,
"command": cmd,
"data": data,
}))
.unwrap_or_else(|e| err_response(cmd, &format!("serialization error: {e}")))
}
fn err_response(cmd: &str, msg: &str) -> String {
serde_json::to_string(&serde_json::json!({
"ok": false,
"command": cmd,
"error": msg,
}))
.unwrap()
}
fn format_addr(addr: &swactor::actor::ActorAddress) -> String {
format!("{addr}")
}
fn full_hex(addr: &swactor::actor::ActorAddress) -> String {
addr.0.iter().map(|b| format!("{b:02x}")).collect()
}
fn enriched_stats(rt: &Runtime, col: &StatsCollector) -> RuntimeStats {
let mut s = rt.stats();
col.enrich(&mut s);
s
}
// ── Commands ────────────────────────────────────────────────────────────
pub fn cmd_help() -> String {
ok_response(
"help",
serde_json::json!({
"commands": [
{"name": "overview", "usage": "overview", "description": "Summary: worker count, actor count, total messages, mailbox depth, panics"},
{"name": "workers", "usage": "workers", "description": "Per-worker stats: actors, mailbox depth, messages, sends (local/cross/inbox), panics"},
{"name": "worker", "usage": "worker <id>", "description": "Single worker detail with tick-phase timing breakdown"},
{"name": "actors", "usage": "actors [--sort mailbox|worker|address] [--limit N] [--worker W]", "description": "List actors with optional sorting, limit, and worker filter"},
{"name": "actor", "usage": "actor <hex_prefix>", "description": "Find actor(s) whose address starts with the given hex prefix"},
{"name": "hot", "usage": "hot [N]", "description": "Top N actors by mailbox depth (default 10)"},
{"name": "phases", "usage": "phases [worker_id]", "description": "Tick-phase time breakdown (all workers or one)"},
{"name": "diff", "usage": "diff <seconds>", "description": "Collect two snapshots N seconds apart, report deltas and rates"},
{"name": "quit", "usage": "quit", "description": "Exit the investigate session"},
]
}),
)
}
pub fn cmd_overview(rt: &Runtime, col: &StatsCollector) -> String {
let stats = enriched_stats(rt, col);
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum();
let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum();
let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum();
let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum();
let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum();
let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum();
ok_response(
"overview",
serde_json::json!({
"workers": stats.num_workers,
"actors": stats.actor_details.len(),
"total_messages_processed": total_msgs,
"total_mailbox_depth": total_mailbox,
"total_panics": total_panics,
"total_type_mismatches": total_type_mismatches,
"sends": {
"local": total_local,
"cross_worker": total_cross,
"inbox": total_inbox,
},
}),
)
}
pub fn cmd_workers(rt: &Runtime) -> String {
let stats = rt.stats();
let workers: Vec<_> = stats
.workers
.iter()
.map(|w| {
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
})
})
.collect();
ok_response("workers", workers)
}
pub fn cmd_worker(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let id: usize = match args.first().and_then(|s| s.parse().ok()) {
Some(id) => id,
None => return err_response("worker", "usage: worker <id>"),
};
let stats = enriched_stats(rt, col);
let w = match stats.workers.iter().find(|w| w.id == id) {
Some(w) => w,
None => {
return err_response(
"worker",
&format!("worker {id} not found (have 0..{})", stats.num_workers),
)
}
};
// Tick phase breakdown for this worker
let timings = stats.tick_timings.get(id).cloned().unwrap_or_default();
let phase_breakdown = compute_phase_breakdown(&timings);
let actors_on_worker: Vec<_> = stats
.actor_details
.iter()
.filter(|a| a.worker_id == id)
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response(
"worker",
serde_json::json!({
"id": w.id,
"actors": w.num_actors,
"mailbox_depth": w.mailbox_depth,
"messages_processed": w.messages_processed,
"local_sends": w.local_sends,
"cross_sends": w.cross_sends,
"inbox_sends": w.inbox_sends,
"type_mismatches": w.type_mismatches,
"panics": w.panics,
"tick_phases": phase_breakdown,
"actor_details": actors_on_worker,
}),
)
}
pub fn cmd_actors(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let stats = enriched_stats(rt, col);
let mut actors = stats.actor_details.clone();
// Parse flags
let mut sort_by = "mailbox";
let mut limit: usize = usize::MAX;
let mut worker_filter: Option<usize> = None;
let mut i = 0;
while i < args.len() {
match args[i] {
"--sort" if i + 1 < args.len() => {
sort_by = args[i + 1];
i += 2;
}
"--limit" if i + 1 < args.len() => {
limit = args[i + 1].parse().unwrap_or(usize::MAX);
i += 2;
}
"--worker" if i + 1 < args.len() => {
worker_filter = args[i + 1].parse().ok();
i += 2;
}
_ => {
i += 1;
}
}
}
if let Some(wid) = worker_filter {
actors.retain(|a| a.worker_id == wid);
}
match sort_by {
"mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)),
"worker" => actors.sort_by_key(|a| a.worker_id),
"address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)),
other => return err_response("actors", &format!("unknown sort field `{other}` — use mailbox|worker|address")),
}
actors.truncate(limit);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response(
"actors",
serde_json::json!({
"total": stats.actor_details.len(),
"returned": rows.len(),
"sort": sort_by,
"actors": rows,
}),
)
}
pub fn cmd_actor(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let prefix = match args.first() {
Some(p) => *p,
None => return err_response("actor", "usage: actor <hex_prefix>"),
};
let stats = enriched_stats(rt, col);
let matches: Vec<_> = stats
.actor_details
.iter()
.filter(|a| full_hex(&a.address).starts_with(prefix))
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response(
"actor",
serde_json::json!({
"prefix": prefix,
"matches": matches.len(),
"actors": matches,
}),
)
}
pub fn cmd_hot(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let n: usize = args.first().and_then(|s| s.parse().ok()).unwrap_or(10);
let stats = enriched_stats(rt, col);
let mut actors = stats.actor_details.clone();
actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth));
actors.truncate(n);
let rows: Vec<_> = actors
.iter()
.map(|a| {
serde_json::json!({
"address": format_addr(&a.address),
"address_full": full_hex(&a.address),
"worker_id": a.worker_id,
"mailbox_depth": a.mailbox_depth,
"last_msg_type": a.last_msg_type,
"messages_processed": a.messages_processed,
"poisoned": a.poisoned,
})
})
.collect();
ok_response("hot", rows)
}
pub fn cmd_phases(rt: &Runtime, args: &[&str]) -> String {
let stats = rt.stats();
let worker_filter: Option<usize> = args.first().and_then(|s| s.parse().ok());
let phase_names = [
"spawn_drain",
"transfer_drain",
"tick_all",
"spawn_drain_2",
"pending_local",
"stats_publish",
];
let mut results = Vec::new();
for (i, timings) in stats.tick_timings.iter().enumerate() {
if let Some(wid) = worker_filter {
if i != wid {
continue;
}
}
let breakdown = compute_phase_breakdown(timings);
results.push(serde_json::json!({
"worker_id": i,
"ticks_sampled": timings.len(),
"phases": breakdown,
"phase_names": phase_names,
}));
}
ok_response("phases", results)
}
pub fn cmd_diff(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
let secs: f64 = match args.first().and_then(|s| s.parse().ok()) {
Some(s) if s > 0.0 && s <= 30.0 => s,
Some(_) => return err_response("diff", "seconds must be between 0 and 30"),
None => return err_response("diff", "usage: diff <seconds>"),
};
let before = enriched_stats(rt, col);
let t0 = Instant::now();
std::thread::sleep(Duration::from_secs_f64(secs));
let after = enriched_stats(rt, col);
let elapsed = t0.elapsed().as_secs_f64();
let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum();
let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum();
let delta_msgs = msgs_after.saturating_sub(msgs_before);
let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum();
let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum();
let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum();
let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum();
let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum();
let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum();
let per_worker: Vec<_> = after
.workers
.iter()
.enumerate()
.map(|(i, w)| {
let prev = before.workers.get(i);
let d = prev
.map(|p| w.messages_processed.saturating_sub(p.messages_processed))
.unwrap_or(0);
serde_json::json!({
"worker_id": i,
"delta_messages": d,
"msg_per_sec": d as f64 / elapsed,
"actors_before": prev.map(|p| p.num_actors).unwrap_or(0),
"actors_after": w.num_actors,
"mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0),
"mailbox_after": w.mailbox_depth,
})
})
.collect();
ok_response(
"diff",
serde_json::json!({
"elapsed_s": elapsed,
"actors_before": before.actor_details.len(),
"actors_after": after.actor_details.len(),
"delta_messages": delta_msgs,
"msg_per_sec": delta_msgs as f64 / elapsed,
"delta_local_sends": local_after.saturating_sub(local_before),
"delta_cross_sends": cross_after.saturating_sub(cross_before),
"mailbox_before": mailbox_before,
"mailbox_after": mailbox_after,
"per_worker": per_worker,
}),
)
}
// ── Phase breakdown helper ──────────────────────────────────────────────
fn compute_phase_breakdown(
timings: &[swactor::stats::TickTiming],
) -> serde_json::Value {
if timings.is_empty() {
return serde_json::json!({
"ticks": 0,
"active_pct": 0.0,
"avg_tick_us": 0.0,
"phases_us": [0, 0, 0, 0, 0, 0],
"phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
});
}
let n = timings.len();
let active = timings.iter().filter(|t| t.did_work).count();
let active_pct = (active as f64 / n as f64) * 100.0;
let mut phase_sums = [0u64; 6];
for t in timings {
for (i, &us) in t.phase_us.iter().enumerate() {
phase_sums[i] += us;
}
}
let total_us: u64 = phase_sums.iter().sum();
let avg_tick_us = total_us as f64 / n as f64;
let phases_pct: Vec<f64> = if total_us == 0 {
vec![0.0; 6]
} else {
phase_sums
.iter()
.map(|&s| (s as f64 / total_us as f64) * 100.0)
.collect()
};
serde_json::json!({
"ticks": n,
"active_pct": active_pct,
"avg_tick_us": avg_tick_us,
"phases_us": phase_sums,
"phases_pct": phases_pct,
})
} }

View file

@ -11,7 +11,6 @@ use swactor::runtime::Runtime;
use crate::actors_html::ACTORS_HTML; use crate::actors_html::ACTORS_HTML;
use crate::collector::StatsCollector; use crate::collector::StatsCollector;
use crate::dashboard_html::DASHBOARD_HTML; use crate::dashboard_html::DASHBOARD_HTML;
use crate::investigate;
use crate::layer::EventStore; use crate::layer::EventStore;
use crate::trace::RuntimeTrace; use crate::trace::RuntimeTrace;
@ -127,6 +126,7 @@ pub(crate) fn spawn_http_server(
let addr = format!("0.0.0.0:{port}"); let addr = format!("0.0.0.0:{port}");
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server"); let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
let server = Arc::new(server); let server = Arc::new(server);
let cmd_router = Arc::new(swactor_command::CommandRouter::with_builtins());
for _ in 0..4 { for _ in 0..4 {
let server = Arc::clone(&server); let server = Arc::clone(&server);
@ -134,6 +134,7 @@ pub(crate) fn spawn_http_server(
let runtime = Arc::clone(&runtime); let runtime = Arc::clone(&runtime);
let collector = Arc::clone(&collector); let collector = Arc::clone(&collector);
let shutdown = Arc::clone(&shutdown); let shutdown = Arc::clone(&shutdown);
let cmd_router = Arc::clone(&cmd_router);
#[cfg(feature = "distribution")] #[cfg(feature = "distribution")]
let distribution = Arc::clone(&distribution); let distribution = Arc::clone(&distribution);
thread::spawn(move || { thread::spawn(move || {
@ -174,6 +175,7 @@ pub(crate) fn spawn_http_server(
&url, &url,
Arc::clone(&runtime), Arc::clone(&runtime),
Arc::clone(&collector), Arc::clone(&collector),
Arc::clone(&cmd_router),
); );
} }
_ => respond_404(request), _ => respond_404(request),
@ -283,21 +285,33 @@ fn handle_investigate_api(
url: &str, url: &str,
runtime: Arc<Mutex<Option<Arc<Runtime>>>>, runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>, collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
cmd_router: Arc<swactor_command::CommandRouter>,
) { ) {
let params = parse_query_string(url); let params = parse_query_string(url);
let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help");
let maybe_rt = runtime.lock().unwrap().clone(); let maybe_rt = runtime.lock().unwrap().clone();
let maybe_col = collector.lock().unwrap().clone(); let maybe_col = collector.lock().unwrap().clone();
let json = match (maybe_rt, maybe_col) { let json = match (maybe_rt, maybe_col) {
(Some(rt), Some(col)) => investigate::dispatch_command(cmd, &params, &rt, &col), (Some(rt), Some(col)) => {
_ => serde_json::json!({ let ctx = swactor_command::CommandContext::with_enricher(rt, col);
let req = swactor_command::from_query_params(&params);
cmd_router.dispatch(&req, &ctx).to_json_line()
}
(Some(rt), None) => {
let ctx = swactor_command::CommandContext::new(rt);
let req = swactor_command::from_query_params(&params);
cmd_router.dispatch(&req, &ctx).to_json_line()
}
_ => {
let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help");
serde_json::json!({
"ok": false, "ok": false,
"command": cmd, "command": cmd,
"error": "runtime not attached yet" "error": "runtime not attached yet"
}) })
.to_string(), .to_string()
}
}; };
let response = tiny_http::Response::from_string(json).with_header( let response = tiny_http::Response::from_string(json).with_header(

View file

@ -73,6 +73,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
swim: config.swim.clone(), swim: config.swim.clone(),
cache_capacity: config.cache_capacity, cache_capacity: config.cache_capacity,
republish_interval: 50, republish_interval: 50,
..Default::default()
}; };
let node = DistributedNode::new(node_config); let node = DistributedNode::new(node_config);
node_ids.push(node.node_id()); node_ids.push(node.node_id());
@ -184,6 +185,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
swim: config.swim.clone(), swim: config.swim.clone(),
cache_capacity: config.cache_capacity, cache_capacity: config.cache_capacity,
republish_interval: 50, republish_interval: 50,
..Default::default()
}; };
let revived = DistributedNode::new(node_config); let revived = DistributedNode::new(node_config);
// Rejoin the cluster. // Rejoin the cluster.

View file

@ -0,0 +1,12 @@
[package]
name = "swactor-wasm-actor"
version = "0.1.0"
edition = "2024"
[dependencies]
swactor = { path = "../.." }
wasmtime = "29"
[dev-dependencies]
swactor = { path = "../..", features = ["getrandom"] }
wat = "1"

View file

@ -0,0 +1,59 @@
use swactor::actor::{ActorInterface, Ctx};
use wasmtime::{Memory, Store, TypedFunc};
use crate::ByteMessage;
/// State accessible to host functions during guest execution.
#[derive(Default)]
pub(crate) struct HostState {
pub outbox: Vec<(swactor::actor::ActorAddress, Vec<u8>)>,
}
/// An actor whose logic is defined by a WebAssembly guest module.
///
/// Messages arrive as [`ByteMessage`], are copied into Wasm linear memory,
/// and processed by the guest's `handle` export. The guest can send messages
/// back via the `swactor.send` host import.
pub struct WasmActor {
pub(crate) store: Store<HostState>,
pub(crate) memory: Memory,
pub(crate) alloc: TypedFunc<i32, i32>,
pub(crate) handle: TypedFunc<(i32, i32), ()>,
}
impl ActorInterface for WasmActor {
type Incoming = ByteMessage;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: ByteMessage) {
let bytes = &msg.0;
let len: i32 = match i32::try_from(bytes.len()) {
Ok(n) => n,
Err(_) => return, // message too large for i32 ABI
};
// 1. Allocate space in guest memory
let ptr = match self.alloc.call(&mut self.store, len) {
Ok(ptr) if ptr < 0 => return, // invalid pointer
Ok(0) if len > 0 => return, // OOM — drop message
Ok(ptr) => ptr,
Err(_) => return, // alloc trapped — drop message
};
// 2. Write message bytes into guest memory
self.memory.data_mut(&mut self.store)
[ptr as usize..(ptr as usize + bytes.len())]
.copy_from_slice(bytes);
// 3. Call guest handle
if self.handle.call(&mut self.store, (ptr, len)).is_err() {
return; // handle trapped — drop message, keep actor alive
}
// 4. Drain outbox → send via ctx
let outbox: Vec<_> = self.store.data_mut().outbox.drain(..).collect();
for (dest, payload) in outbox {
let _ = ctx.send(dest, ByteMessage(payload));
}
}
}

View file

@ -0,0 +1,109 @@
use swactor::actor::ActorAddress;
use wasmtime::{Linker, Module, Store, TypedFunc};
use crate::actor::{HostState, WasmActor};
use crate::engine::SharedEngine;
use crate::error::WasmActorError;
/// Compiles a Wasm module and produces a ready-to-use [`WasmActor`].
pub struct WasmActorBuilder {
engine: SharedEngine,
wasm_bytes: Vec<u8>,
}
impl WasmActorBuilder {
pub fn new(engine: SharedEngine, wasm_bytes: impl Into<Vec<u8>>) -> Self {
Self {
engine,
wasm_bytes: wasm_bytes.into(),
}
}
/// Compile the module, link host functions, and instantiate.
pub fn build(self) -> Result<WasmActor, WasmActorError> {
let engine = self.engine.inner();
let module = Module::new(engine, &self.wasm_bytes)?;
let mut linker: Linker<HostState> = Linker::new(engine);
Self::link_send(&mut linker)?;
let mut store = Store::new(engine, HostState::default());
let instance = linker.instantiate(&mut store, &module)?;
// Extract required exports
let memory = instance
.get_memory(&mut store, "memory")
.ok_or(WasmActorError::MissingExport("memory"))?;
let alloc: TypedFunc<i32, i32> = instance
.get_typed_func(&mut store, "alloc")
.map_err(|_| WasmActorError::MissingExport("alloc"))?;
let handle: TypedFunc<(i32, i32), ()> = instance
.get_typed_func(&mut store, "handle")
.map_err(|_| WasmActorError::MissingExport("handle"))?;
Ok(WasmActor {
store,
memory,
alloc,
handle,
})
}
/// Link the `swactor.send` host import.
fn link_send(linker: &mut Linker<HostState>) -> Result<(), WasmActorError> {
linker.func_wrap(
"swactor",
"send",
|mut caller: wasmtime::Caller<'_, HostState>,
dest_ptr: i32,
payload_ptr: i32,
payload_len: i32|
-> Result<(), wasmtime::Error> {
let mem = caller
.get_export("memory")
.and_then(|e| e.into_memory())
.ok_or_else(|| wasmtime::Error::msg("guest must export memory"))?;
let data = mem.data(&caller);
let mem_len = data.len();
// Validate non-negative arguments
if dest_ptr < 0 || payload_ptr < 0 || payload_len < 0 {
return Err(wasmtime::Error::msg(
"negative argument in swactor.send",
));
}
let dest_ptr = dest_ptr as usize;
let payload_ptr = payload_ptr as usize;
let payload_len = payload_len as usize;
// Bounds-check with overflow protection
let dest_end = dest_ptr
.checked_add(32)
.ok_or_else(|| wasmtime::Error::msg("dest_ptr overflow"))?;
let payload_end = payload_ptr
.checked_add(payload_len)
.ok_or_else(|| wasmtime::Error::msg("payload range overflow"))?;
if dest_end > mem_len || payload_end > mem_len {
return Err(wasmtime::Error::msg(
"out-of-bounds memory access in swactor.send",
));
}
// Read 32-byte destination address
let mut addr_bytes = [0u8; 32];
addr_bytes.copy_from_slice(&data[dest_ptr..dest_end]);
let dest = ActorAddress(addr_bytes);
// Read payload
let payload = data[payload_ptr..payload_end].to_vec();
caller.data_mut().outbox.push((dest, payload));
Ok(())
},
)?;
Ok(())
}
}

View file

@ -0,0 +1,35 @@
use std::sync::Arc;
use wasmtime::Engine;
/// A shared, cheaply-cloneable Wasm engine.
///
/// Created once and reused across multiple [`WasmActor`](crate::WasmActor) instances.
/// Configured with maximum sandboxing — no threads, no SIMD, no reference types.
#[derive(Clone)]
pub struct SharedEngine(Arc<Engine>);
impl std::fmt::Debug for SharedEngine {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("SharedEngine").field(&"<Engine>").finish()
}
}
impl SharedEngine {
/// Create a new engine with sandboxed defaults.
pub fn new() -> Result<Self, wasmtime::Error> {
let mut config = wasmtime::Config::new();
config.wasm_threads(false);
config.wasm_simd(false);
config.wasm_relaxed_simd(false);
config.wasm_reference_types(false);
config.wasm_multi_value(false);
config.wasm_bulk_memory(true);
let engine = Engine::new(&config)?;
Ok(Self(Arc::new(engine)))
}
pub(crate) fn inner(&self) -> &Engine {
&self.0
}
}

View file

@ -0,0 +1,27 @@
use std::fmt;
/// Errors that can occur when building or running a WasmActor.
#[derive(Debug)]
pub enum WasmActorError {
/// A required export is missing from the Wasm module.
MissingExport(&'static str),
/// The Wasm module failed to compile or instantiate.
Wasmtime(wasmtime::Error),
}
impl fmt::Display for WasmActorError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::MissingExport(name) => write!(f, "missing required export: `{name}`"),
Self::Wasmtime(e) => write!(f, "wasmtime error: {e}"),
}
}
}
impl std::error::Error for WasmActorError {}
impl From<wasmtime::Error> for WasmActorError {
fn from(e: wasmtime::Error) -> Self {
Self::Wasmtime(e)
}
}

View file

@ -0,0 +1,13 @@
mod actor;
mod builder;
mod engine;
mod error;
pub use actor::WasmActor;
pub use builder::WasmActorBuilder;
pub use engine::SharedEngine;
pub use error::WasmActorError;
/// A message carrying raw bytes, suitable for passing to/from Wasm guests.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ByteMessage(pub Vec<u8>);

View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "double-guest"
version = "0.1.0"

View file

@ -0,0 +1,13 @@
[workspace]
[package]
name = "double-guest"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["cdylib"]
[profile.release]
opt-level = "s"
lto = true

View file

@ -0,0 +1,63 @@
#![no_std]
use core::cell::UnsafeCell;
use core::panic::PanicInfo;
// --- bump allocator ---
const HEAP_SIZE: usize = 65536;
struct BumpAlloc {
heap: UnsafeCell<[u8; HEAP_SIZE]>,
offset: UnsafeCell<usize>,
}
unsafe impl Sync for BumpAlloc {}
static ALLOC: BumpAlloc = BumpAlloc {
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
offset: UnsafeCell::new(0),
};
#[unsafe(no_mangle)]
pub extern "C" fn alloc(size: i32) -> i32 {
unsafe {
let offset = &mut *ALLOC.offset.get();
let heap = &mut *ALLOC.heap.get();
let align = 8;
let start = (*offset + align - 1) & !(align - 1);
let end = start + size as usize;
if end > heap.len() {
return 0; // OOM
}
*offset = end;
heap.as_ptr().add(start) as i32
}
}
// --- host import ---
#[link(wasm_import_module = "swactor")]
unsafe extern "C" {
#[link_name = "send"]
fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32);
}
/// Message format: first 32 bytes = destination address, rest = payload.
/// Sends the payload back twice to demonstrate multi-send.
#[unsafe(no_mangle)]
pub extern "C" fn handle(ptr: i32, len: i32) {
if len < 32 {
return;
}
let dest_ptr = ptr;
let payload_ptr = ptr + 32;
let payload_len = len - 32;
unsafe {
host_send(dest_ptr, payload_ptr, payload_len);
host_send(dest_ptr, payload_ptr, payload_len);
}
}
#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
loop {}
}

View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "echo-guest"
version = "0.1.0"

View file

@ -0,0 +1,13 @@
[workspace]
[package]
name = "echo-guest"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["cdylib"]
[profile.release]
opt-level = "s"
lto = true

View file

@ -0,0 +1,62 @@
#![no_std]
use core::cell::UnsafeCell;
use core::panic::PanicInfo;
// --- bump allocator ---
const HEAP_SIZE: usize = 65536;
struct BumpAlloc {
heap: UnsafeCell<[u8; HEAP_SIZE]>,
offset: UnsafeCell<usize>,
}
unsafe impl Sync for BumpAlloc {}
static ALLOC: BumpAlloc = BumpAlloc {
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
offset: UnsafeCell::new(0),
};
#[unsafe(no_mangle)]
pub extern "C" fn alloc(size: i32) -> i32 {
unsafe {
let offset = &mut *ALLOC.offset.get();
let heap = &mut *ALLOC.heap.get();
let align = 8;
let start = (*offset + align - 1) & !(align - 1);
let end = start + size as usize;
if end > heap.len() {
return 0; // OOM
}
*offset = end;
heap.as_ptr().add(start) as i32
}
}
// --- host import ---
#[link(wasm_import_module = "swactor")]
unsafe extern "C" {
#[link_name = "send"]
fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32);
}
/// Message format: first 32 bytes = destination address, rest = payload.
/// Echo sends the payload portion back to the specified destination.
#[unsafe(no_mangle)]
pub extern "C" fn handle(ptr: i32, len: i32) {
if len < 32 {
return;
}
let dest_ptr = ptr;
let payload_ptr = ptr + 32;
let payload_len = len - 32;
unsafe {
host_send(dest_ptr, payload_ptr, payload_len);
}
}
#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
loop {}
}

View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "silent-guest"
version = "0.1.0"

View file

@ -0,0 +1,13 @@
[workspace]
[package]
name = "silent-guest"
version = "0.1.0"
edition = "2024"
[lib]
crate-type = ["cdylib"]
[profile.release]
opt-level = "s"
lto = true

View file

@ -0,0 +1,45 @@
#![no_std]
use core::cell::UnsafeCell;
use core::panic::PanicInfo;
// --- bump allocator ---
const HEAP_SIZE: usize = 65536;
struct BumpAlloc {
heap: UnsafeCell<[u8; HEAP_SIZE]>,
offset: UnsafeCell<usize>,
}
unsafe impl Sync for BumpAlloc {}
static ALLOC: BumpAlloc = BumpAlloc {
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
offset: UnsafeCell::new(0),
};
#[unsafe(no_mangle)]
pub extern "C" fn alloc(size: i32) -> i32 {
unsafe {
let offset = &mut *ALLOC.offset.get();
let heap = &mut *ALLOC.heap.get();
let align = 8;
let start = (*offset + align - 1) & !(align - 1);
let end = start + size as usize;
if end > heap.len() {
return 0; // OOM
}
*offset = end;
heap.as_ptr().add(start) as i32
}
}
#[unsafe(no_mangle)]
pub extern "C" fn handle(_ptr: i32, _len: i32) {
// Silent: receive bytes, do nothing
}
#[panic_handler]
fn panic(_info: &PanicInfo) -> ! {
loop {}
}

View file

@ -0,0 +1,331 @@
use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder, WasmActorError};
fn guest_wasm(name: &str) -> Vec<u8> {
let path = format!(
"{}/tests/guests/{name}/target/wasm32-unknown-unknown/release/{name}_guest.wasm",
env!("CARGO_MANIFEST_DIR")
);
std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {path}: {e}"))
}
/// Build a message with an inbox address prepended (the guest contract).
fn framed_msg(dest: &ActorAddress, payload: &[u8]) -> ByteMessage {
let mut buf = Vec::with_capacity(32 + payload.len());
buf.extend_from_slice(&dest.0);
buf.extend_from_slice(payload);
ByteMessage(buf)
}
// ── Echo: send bytes in, same bytes come back ────────────────────────────────
#[test]
fn echo_returns_same_payload() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("echo"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
let payload = b"hello wasm";
rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap();
rt.tick();
let received = inbox.try_recv().expect("inbox should have a message");
assert_eq!(received.0, payload);
}
#[test]
fn echo_preserves_binary_payload() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("echo"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
let payload: Vec<u8> = (0..=255).collect();
rt.send_to(addr, framed_msg(inbox.addr(), &payload)).unwrap();
rt.tick();
let received = inbox.try_recv().expect("inbox should have a message");
assert_eq!(received.0, payload);
}
// ── Silent: processes messages without sending anything ───────────────────────
#[test]
fn silent_produces_no_output() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("silent"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(b"ignored".to_vec())).unwrap();
rt.tick();
assert!(inbox.try_recv().is_none(), "silent guest should not send anything");
}
// ── Double: one message in, two messages out ─────────────────────────────────
#[test]
fn double_sends_two_copies() {
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, guest_wasm("double"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
let payload = b"dup me";
rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap();
rt.tick();
let first = inbox.try_recv().expect("should receive first copy");
let second = inbox.try_recv().expect("should receive second copy");
assert_eq!(first.0, payload);
assert_eq!(second.0, payload);
assert!(inbox.try_recv().is_none(), "exactly two messages expected");
}
// ── Missing export → WasmActorError::MissingExport ───────────────────────────
#[test]
fn missing_alloc_export_returns_error() {
// Minimal valid Wasm module: (module) — no exports at all
let minimal_wasm = wat::parse_str("(module)").unwrap();
let engine = SharedEngine::new().unwrap();
let result = WasmActorBuilder::new(engine, minimal_wasm).build();
match result {
Err(WasmActorError::MissingExport(name)) => {
assert!(
name == "memory" || name == "alloc",
"expected missing memory or alloc, got: {name}"
);
}
Err(other) => panic!("expected MissingExport, got: {other}"),
Ok(_) => panic!("expected error for module with no exports"),
}
}
// ── Engine sharing: two actors from the same engine ──────────────────────────
#[test]
fn shared_engine_serves_multiple_actors() {
let engine = SharedEngine::new().unwrap();
let echo = WasmActorBuilder::new(engine.clone(), guest_wasm("echo"))
.build()
.unwrap();
let silent = WasmActorBuilder::new(engine, guest_wasm("silent"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let echo_addr = rt.spawn(echo).unwrap();
let _silent_addr = rt.spawn(silent).unwrap();
let payload = b"shared engine test";
rt.send_to(echo_addr, framed_msg(inbox.addr(), payload)).unwrap();
rt.tick();
let received = inbox.try_recv().expect("echo actor should still work");
assert_eq!(received.0, payload);
}
// ── Safety: edge cases that previously caused panics or corruption ────────────
#[test]
fn oob_send_traps_cleanly_and_actor_survives() {
// Guest calls swactor.send with dest_ptr pointing past the end of memory.
// The host should trap the call; the actor should survive for future messages.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const 0 ;; return start of memory (simplistic)
)
(func (export "handle") (param i32 i32)
;; Call send with dest_ptr = 65536 (1 page = end of memory, OOB for 32 bytes)
i32.const 65536
i32.const 0
i32.const 0
call $send
)
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let addr = rt.spawn(actor).unwrap();
// Send a message — handle will try OOB send, which traps
rt.send_to(addr, ByteMessage(vec![42])).unwrap();
rt.tick();
// No message should arrive (the send was invalid)
assert!(inbox.try_recv().is_none(), "OOB send should not produce a message");
}
#[test]
fn alloc_oom_drops_message_actor_stays_alive() {
// Guest alloc always returns 0 (OOM). Message should be dropped,
// actor should remain alive for subsequent messages.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const 0 ;; always OOM
)
(func (export "handle") (param i32 i32)
;; Should never be called if alloc returned 0 for non-zero len
)
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
// Send a non-empty message — alloc returns 0, message should be dropped
rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap();
rt.tick();
// Actor is still alive — send another message, tick again (no panic)
rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap();
rt.tick();
}
#[test]
fn negative_alloc_ptr_drops_message() {
// Guest alloc returns -1. Host should detect the negative pointer and drop.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const -1 ;; invalid negative pointer
)
(func (export "handle") (param i32 i32))
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(vec![1])).unwrap();
rt.tick(); // should not panic
// Actor survives
rt.send_to(addr, ByteMessage(vec![2])).unwrap();
rt.tick();
}
#[test]
fn handle_trap_drops_message_actor_survives() {
// Guest handle executes `unreachable`, causing a Wasm trap.
// Message should be dropped, actor should stay alive.
let wat = r#"
(module
(import "swactor" "send" (func $send (param i32 i32 i32)))
(memory (export "memory") 1)
(func (export "alloc") (param i32) (result i32)
i32.const 256 ;; valid allocation
)
(func (export "handle") (param i32 i32)
unreachable ;; trap!
)
)
"#;
let wasm = wat::parse_str(wat).unwrap();
let engine = SharedEngine::new().unwrap();
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap();
rt.tick(); // handle traps, but actor should survive
// Actor is still alive
rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap();
rt.tick();
}
// ── Integration: WasmActor alongside a native Rust actor ─────────────────────
#[derive(Clone)]
struct ForwardToWasm {
wasm_addr: ActorAddress,
inbox_addr: ActorAddress,
}
struct Forwarder;
impl ActorInterface for Forwarder {
type Incoming = ForwardToWasm;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: ForwardToWasm) {
// Build the framed message and forward to the wasm actor
let payload = b"from native";
let framed = framed_msg(&msg.inbox_addr, payload);
let _ = ctx.send(msg.wasm_addr, framed);
}
}
#[test]
fn native_actor_communicates_with_wasm_actor() {
let engine = SharedEngine::new().unwrap();
let wasm = WasmActorBuilder::new(engine, guest_wasm("echo"))
.build()
.unwrap();
let rt = Runtime::new(RuntimeConfig::default());
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
let wasm_addr = rt.spawn(wasm).unwrap();
let forwarder_addr = rt.spawn(Forwarder).unwrap();
rt.send_to(
forwarder_addr,
ForwardToWasm {
wasm_addr,
inbox_addr: *inbox.addr(),
},
)
.unwrap();
// Tick 1: Forwarder receives message and sends to WasmActor
rt.tick();
// Tick 2: WasmActor receives the forwarded message and echoes to inbox
rt.tick();
let received = inbox.try_recv().expect("wasm actor should have echoed");
assert_eq!(received.0, b"from native");
}

View file

@ -0,0 +1,191 @@
# Wasm Actor Crate — Development History
> Adds a new crate (`crates/wasm-actor/`) that runs WebAssembly guest code
> **inside** a swactor actor. The Wasm instance lives in the actor — not as a
> separate OS process. Messages arrive as bytes, get written into Wasm linear
> memory, and the guest's `handle` export is called.
>
> ~350 lines of Rust (host) · 3 guest modules · 7 tests
---
## Table of Contents
1. [Overview & Motivation](#1-overview--motivation)
2. [What Was Built](#2-what-was-built)
3. [Guest ↔ Host Contract](#3-guest--host-contract)
4. [Handle Cycle (Hot Path)](#4-handle-cycle-hot-path)
5. [Guest Modules](#5-guest-modules)
6. [Design Decisions & Tradeoffs](#6-design-decisions--tradeoffs)
7. [Known Gaps & Future Improvements](#7-known-gaps--future-improvements)
8. [Test Coverage Summary](#8-test-coverage-summary)
---
## 1. Overview & Motivation
Swactor already supported running *inside* a browser via `crates/wasm/`
(wasm-bindgen). This crate flips the direction: run untrusted Wasm code
*inside* an actor, sandboxed by wasmtime. Use cases include user-defined
plugins, multi-language actors, and capability-restricted compute.
The main swactor crate has no wasmtime dependency — all Wasm machinery is
isolated in `crates/wasm-actor/`.
---
## 2. What Was Built
| Component | Location | Purpose |
|-----------|----------|---------|
| `swactor-wasm-actor` crate | `crates/wasm-actor/` | Host-side: engine, builder, actor impl |
| 3 guest crates | `crates/wasm-actor/tests/guests/{echo,double,silent}/` | `#![no_std]` Wasm modules for testing |
| Integration tests | `crates/wasm-actor/tests/wasm_actor.rs` | 7 behavioral tests |
### Crate modules
```
crates/wasm-actor/src/
lib.rs — ByteMessage, re-exports
engine.rs — SharedEngine (Arc<wasmtime::Engine>)
builder.rs — WasmActorBuilder (compile + link + instantiate)
actor.rs — WasmActor implementing ActorInterface
error.rs — WasmActorError enum
```
### Public types
- **`ByteMessage(pub Vec<u8>)`** — message type for Wasm actors. Satisfies
`Message` bounds trivially.
- **`SharedEngine`** — wraps `Arc<wasmtime::Engine>`. Created once, cloned
cheaply across actors. Sandboxed config: no threads, no SIMD, no reference
types.
- **`WasmActorBuilder`** — takes an engine + raw `.wasm` bytes, compiles the
module, links the `swactor.send` host import, extracts typed function handles,
returns a `WasmActor`.
- **`WasmActor`** — implements `ActorInterface<Incoming = ByteMessage, Response = ()>`.
- **`WasmActorError`** — `MissingExport(&'static str)` or `Wasmtime(wasmtime::Error)`.
---
## 3. Guest ↔ Host Contract
**Guest must export:**
| Export | Signature | Purpose |
|--------|-----------|---------|
| `memory` | WebAssembly linear memory | Host reads/writes message bytes here |
| `alloc` | `(size: i32) -> i32` | Allocate `size` bytes, return pointer |
| `handle` | `(ptr: i32, len: i32)` | Process message at `(ptr, len)` |
**Guest may import:**
| Import | Module | Signature | Purpose |
|--------|--------|-----------|---------|
| `send` | `swactor` | `(dest_ptr: i32, payload_ptr: i32, payload_len: i32)` | Send a message to another actor |
`dest_ptr` points to 32 bytes of `ActorAddress` in guest linear memory.
`payload_ptr` + `payload_len` describe the message bytes.
---
## 4. Handle Cycle (Hot Path)
```
ByteMessage arrives
│
v
1. host calls guest alloc(msg.len) → ptr
│
v
2. host writes msg bytes into guest memory at ptr
│
v
3. host calls guest handle(ptr, len)
│
├── guest may call swactor.send() N times
│ └── each appends (ActorAddress, Vec<u8>) to HostState.outbox
│
v
4. host drains outbox → ctx.send(dest, ByteMessage(payload)) for each
```
Traps during `alloc` or `handle` will panic. Swactor's existing
`catch_unwind` in `tick_all` poisons the actor — consistent with the
panic-safety model.
---
## 5. Guest Modules
Three `#![no_std]` Rust crates compiled to `wasm32-unknown-unknown`:
| Guest | Behavior | Tests it supports |
|-------|----------|-------------------|
| `echo` | Reads 32-byte dest + payload from message; sends payload back to dest | Echo roundtrip, binary preservation |
| `double` | Same framing; sends payload back **twice** | Multi-send verification |
| `silent` | Receives bytes; does nothing | No-output / no-error baseline |
Each guest uses a simple inline bump allocator (64 KiB heap, 8-byte aligned)
and a `#[panic_handler]` that loops. No external dependencies.
Message framing convention: the first 32 bytes of the `ByteMessage` payload
are the destination `ActorAddress`, followed by the actual message bytes.
This allows guests to send replies without hardcoding addresses.
### Building guests
```bash
rustup target add wasm32-unknown-unknown # one-time
cd crates/wasm-actor/tests/guests/echo && cargo build --target wasm32-unknown-unknown --release
cd crates/wasm-actor/tests/guests/double && cargo build --target wasm32-unknown-unknown --release
cd crates/wasm-actor/tests/guests/silent && cargo build --target wasm32-unknown-unknown --release
```
Each guest crate has its own `[workspace]` marker to stay independent of the
root workspace.
---
## 6. Design Decisions & Tradeoffs
| # | Decision | Rationale |
|---|----------|-----------|
| 1 | **wasmtime, not wasmer/wasm3** | Best-maintained, fuel metering support, cranelift JIT |
| 2 | **Raw bytes, not structured messages** | Keeps the boundary simple; framing/serialization is the guest's concern |
| 3 | **Separate crate, not a feature flag** | wasmtime is ~30 crates; most users don't need it in their dependency tree |
| 4 | **Bump allocator in guests** | Zero-dependency, predictable, sufficient for request/response patterns |
| 5 | **Dest address in message payload** | Avoids hardcoded addresses; guests can send to any actor the host tells them about |
| 6 | **Traps = panics (no Result)** | Matches swactor's existing panic-safety model; `catch_unwind` in `tick_all` poisons the actor |
| 7 | **Engine sharing via Arc** | Module compilation is expensive; `SharedEngine` amortizes it across actors |
| 8 | **Maximum sandboxing defaults** | Disabled: threads, SIMD, relaxed SIMD, reference types, multi-value. Enabled: bulk memory (required by most compilers) |
---
## 7. Known Gaps & Future Improvements
| # | Gap | Notes |
|---|-----|-------|
| 1 | **No fuel metering** | wasmtime supports fuel; maps naturally to per-tick actor budgets. Deferred to follow-up. |
| 2 | **No WASI** | No filesystem, network, random, or clock access. Intentional for sandboxing, but limits guest capabilities. |
| 3 | **No guest SDK crate** | The test guests serve as examples. A published `swactor-guest` crate with the alloc/handle/send glue would reduce boilerplate. |
| 4 | **Bump allocator never frees** | Fine for short-lived handle calls, but long-running actors would need a real allocator. |
| 5 | **No pre-compilation cache** | `Module::new()` recompiles every time. wasmtime supports serialized modules for faster cold starts. |
| 6 | **`cargo test -p` doesn't resolve** | Must use `--manifest-path`. Workspace resolution quirk. |
---
## 8. Test Coverage Summary
7 behavioral tests in `crates/wasm-actor/tests/wasm_actor.rs`:
| Test | Scenario |
|------|----------|
| `echo_returns_same_payload` | Send bytes → wasm echoes them back to inbox |
| `echo_preserves_binary_payload` | All 256 byte values survive the roundtrip |
| `silent_produces_no_output` | Guest does nothing; no error, no messages |
| `double_sends_two_copies` | One message in → two messages out |
| `missing_alloc_export_returns_error` | WAT module with no exports → `WasmActorError::MissingExport` |
| `shared_engine_serves_multiple_actors` | Two actors from the same `SharedEngine` work independently |
| `native_actor_communicates_with_wasm_actor` | Native Rust actor → WasmActor → inbox (two-tick delivery) |

View file

@ -0,0 +1,335 @@
# Actor Watching — Distributed Death Notifications
## Problem
When an actor dies (panic, explicit stop, or its host node leaves the cluster), other actors that depend on it have no way to know. This is the distributed equivalent of `waitpid()` / `SIGCHLD` — the foundational primitive for building supervision, reconnection logic, and self-healing.
## Design
### Types
```rust
// src/actor.rs
/// Why an actor exited.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ExitReason {
/// Actor was explicitly stopped or removed from the pool.
Stopped,
/// Actor panicked during message handling.
Panicked,
/// The node hosting the actor left the cluster (SWIM Dead).
NodeDown,
}
/// Delivered to watchers when a watched actor exits.
///
/// Implements `Message` (Clone + Send + Sync + 'static) so it can be
/// delivered through normal mailbox channels.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ActorExited {
/// The address of the actor that died.
pub addr: ActorAddress,
/// Why it exited.
pub reason: ExitReason,
}
```
### API
```rust
// src/actor.rs — extend ContextInner
pub trait ContextInner {
// existing:
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error>;
fn spawn_any(&self, addr: ActorAddress, actor: Box<dyn AnyActor>);
// new:
fn watch(&self, watcher: ActorAddress, target: ActorAddress);
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress);
}
// src/actor.rs — extend Ctx
impl Ctx<'_> {
/// Watch another actor's liveness. If the target dies, this actor
/// receives an `ActorExited` message in its mailbox.
///
/// Watching an already-dead or non-existent actor delivers
/// `ActorExited { reason: Stopped }` immediately (on next tick).
///
/// Calling watch() multiple times on the same target is idempotent —
/// only one notification is delivered.
pub fn watch(&self, target: ActorAddress) {
self.inner.watch(self.self_addr, target);
}
/// Stop watching an actor. No notification will be delivered if the
/// target subsequently dies.
pub fn unwatch(&self, target: ActorAddress) {
self.inner.unwatch(self.self_addr, target);
}
}
```
### WatchRegistry
A per-worker structure that tracks watch relationships:
```rust
// src/worker.rs
struct WatchRegistry {
/// target -> set of watchers awaiting death notification
watchers: HashMap<ActorAddress, HashSet<ActorAddress>>,
/// watcher -> set of targets it's watching (reverse index for cleanup)
watching: HashMap<ActorAddress, HashSet<ActorAddress>>,
}
impl WatchRegistry {
fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) {
self.watchers.entry(target).or_default().insert(watcher);
self.watching.entry(watcher).or_default().insert(target);
}
fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) {
if let Some(set) = self.watchers.get_mut(&target) {
set.remove(&watcher);
if set.is_empty() { self.watchers.remove(&target); }
}
if let Some(set) = self.watching.get_mut(&watcher) {
set.remove(&target);
if set.is_empty() { self.watching.remove(&watcher); }
}
}
/// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs.
fn notify_death(&mut self, target: ActorAddress, reason: ExitReason)
-> Vec<(ActorAddress, ActorExited)>
{
let notification = ActorExited { addr: target, reason };
let mut result = Vec::new();
if let Some(watcher_set) = self.watchers.remove(&target) {
for watcher in &watcher_set {
result.push((*watcher, notification.clone()));
// clean up reverse index
if let Some(set) = self.watching.get_mut(watcher) {
set.remove(&target);
if set.is_empty() { self.watching.remove(watcher); }
}
}
}
result
}
/// Called when a watcher itself dies. Cleans up all its watching entries.
fn cleanup_watcher(&mut self, watcher: &ActorAddress) {
if let Some(targets) = self.watching.remove(watcher) {
for target in targets {
if let Some(set) = self.watchers.get_mut(&target) {
set.remove(watcher);
if set.is_empty() { self.watchers.remove(&target); }
}
}
}
}
}
```
### Integration with tick_once
The worker's `tick_once` gains **two new mechanisms**:
**A. Watch/unwatch request processing** — `WorkerContext::watch()` needs to route the request to the correct worker (the one that owns the target). This mirrors how sends work:
```
watch(watcher_A, target_B) called on worker 0:
- target_B is on worker 0? → register locally in WatchRegistry
- target_B is on worker 1? → send WatchCommand through transfer queue
- target_B not in address_map? → may be remote (see Remote Watches below)
```
New envelope variant for internal watch commands:
```rust
// src/delivery.rs
enum InternalCommand {
Watch { watcher: ActorAddress, target: ActorAddress },
Unwatch { watcher: ActorAddress, target: ActorAddress },
}
```
These are delivered through the existing transfer queue alongside `Envelope`s. The transfer queue type becomes `enum TransferItem { Message(Envelope), Command(InternalCommand) }`, or — simpler — the WatchRegistry is shared (behind Arc<Mutex>) and watch/unwatch are applied directly. The shared approach is better since watches are rare relative to messages.
**Recommended**: `Arc<Mutex<WatchRegistry>>` shared across workers, owned by Runtime. Workers hold a reference. Contention is negligible because watch/unwatch operations are rare.
**B. Death notification dispatch** — added to tick_all's panic detection:
```rust
// In ActorPool::tick_all, after catching a panic:
Err(_) => {
slot.poisoned = true;
slot.mailbox.clear();
// NEW: collect death notification
deaths.push((addr, ExitReason::Panicked));
}
```
After tick_all completes, the worker processes `deaths`:
```rust
// In tick_once, after tick_all:
for (addr, reason) in deaths {
let notifications = watch_registry.lock().notify_death(addr, reason);
for (watcher_addr, msg) in notifications {
// Deliver ActorExited as a normal message
self.deliver_to(watcher_addr, Box::new(msg), tc);
}
// Also clean up the dead actor's own watches
watch_registry.lock().cleanup_watcher(&addr);
}
```
### Watching Non-Existent Actors
If `watch(watcher, target)` is called and `target` doesn't exist in the address map:
- **Local runtime**: deliver `ActorExited { reason: Stopped }` immediately (on next tick). The actor is already gone.
- **Distributed**: the watch request is forwarded to the node that should own the target (via Kademlia resolution). If the target doesn't exist there either, a `ActorExitedNotify` is sent back.
### Remote Watches (cross-node)
Wire protocol additions in `crates/distribution/src/messages.rs`:
```rust
/// Request from node A to node B: "notify me if this actor dies"
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WatchRequest {
pub watcher_node: NodeId,
pub watcher_addr: ActorAddress,
pub target_addr: ActorAddress,
}
impl NetworkMessage for WatchRequest {
fn type_tag() -> &'static str { "swactor_dist::WatchRequest" }
}
/// Request to cancel a remote watch
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UnwatchRequest {
pub watcher_node: NodeId,
pub watcher_addr: ActorAddress,
pub target_addr: ActorAddress,
}
impl NetworkMessage for UnwatchRequest {
fn type_tag() -> &'static str { "swactor_dist::UnwatchRequest" }
}
/// Notification from target's node to watcher's node
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ActorExitedNotify {
pub target_addr: ActorAddress,
pub reason: ExitReason,
/// Which watchers on the receiving node should be notified
pub watchers: Vec<ActorAddress>,
}
impl NetworkMessage for ActorExitedNotify {
fn type_tag() -> &'static str { "swactor_dist::ActorExitedNotify" }
}
```
**Flow**:
```
Node A Node B
│ │
│ ctx.watch(watcher_A, target_B) │
│ │
│ ── WatchRequest ──────────────► │
│ │ registers remote watch:
│ │ target_B → (NodeA, watcher_A)
│ │
│ ... time passes ... │
│ │
│ │ target_B panics
│ │
│ ◄── ActorExitedNotify ──────── │
│ │
│ delivers ActorExited to │
│ watcher_A's mailbox │
```
The `WatchRegistry` on node B stores remote watches with the additional `NodeId` of the watcher's node. On death, it partitions notifications into local (deliver directly) and remote (send `ActorExitedNotify` to the watcher's node).
### SWIM Integration
When `handle_membership_change` detects `MemberState::Dead` (`crates/distribution/src/node.rs:269`):
1. The node maintains a **node actor index**: `NodeId -> Set<ActorAddress>` — all actors known to be on each node. This is populated from:
- Directory entries stored locally
- Cache entries
- Remote watch registrations
2. On node death:
```rust
fn handle_membership_change(&mut self, node_id: NodeId, state: MemberState) {
match state {
MemberState::Dead => {
// existing cleanup...
self.routing_table.remove(&node_id);
self.cache.invalidate_node(&node_id);
self.repair_queue.on_node_death(&node_id, &mut self.directory);
// NEW: notify local watchers about all actors on the dead node
if let Some(actor_addrs) = self.node_actor_index.remove(&node_id) {
for actor_addr in actor_addrs {
let notifications = self.watch_registry.notify_death(
actor_addr,
ExitReason::NodeDown,
);
// Queue for delivery to local actors
self.pending_exit_notifications.extend(notifications);
}
}
}
// ...
}
}
```
3. The `pending_exit_notifications` are drained by the NodeDriver on the next tick and delivered into the local runtime.
### Edge Cases
| Scenario | Behavior |
|---|---|
| Watch self | Allowed. On death, ActorExited delivered to own mailbox (no-op since dead). |
| Watch already-dead actor | `ActorExited { reason: Stopped }` delivered on next tick. |
| Watcher dies before target | Cleanup removes all watching entries. No notification delivered. |
| Target node suspected (not yet dead) | No notification — wait for SWIM to confirm Dead or Alive. |
| Network partition heals | If target was falsely declared Dead, a stale `NodeDown` was sent. The watcher may re-watch. No automatic "un-death" notification. |
| Double watch | Idempotent — only one notification per death event. |
### Files Modified
| File | Change |
|------|--------|
| `src/actor.rs` | `ExitReason`, `ActorExited`, `watch()`/`unwatch()` on `ContextInner`, `Ctx` |
| `src/worker.rs` | `WatchRegistry`, death collection in `tick_all`, notification dispatch in `tick_once` |
| `src/runtime.rs` | `Arc<Mutex<WatchRegistry>>` owned by Runtime, passed to workers |
| `crates/distribution/src/messages.rs` | `WatchRequest`, `UnwatchRequest`, `ActorExitedNotify` |
| `crates/distribution/src/node.rs` | Node actor index, SWIM Dead fan-out, `pending_exit_notifications` |
### Tests
- **watch_local_death**: spawn watcher + target, kill target (panic), verify watcher receives `ActorExited { reason: Panicked }`
- **unwatch_prevents_notification**: watch then unwatch, kill target, verify no notification
- **watch_nonexistent**: watch an address that was never spawned, verify `ActorExited { reason: Stopped }`
- **watcher_dies_first**: watch target, kill watcher, kill target — no panic/leak
- **cross_worker_watch**: target on worker 0, watcher on worker 1, kill target, verify notification arrives
- **idempotent_watch**: watch same target twice, kill target, verify exactly one notification

View file

@ -0,0 +1,321 @@
# Cluster-Wide Registry — Distributed Naming
## Problem
Actors can only be found by their `ActorAddress` (a random 32-byte ID). The local `AddressMap` maps addresses to workers on a single node. The Kademlia directory maps addresses to `NodeId`. But neither provides **human-readable naming** or **re-discovery after churn**.
When a node dies and an actor is re-spawned elsewhere, it gets a new `ActorAddress`. Without a name-based registry, every actor that communicated with it needs manual reconfiguration. This doesn't work for churning infrastructure.
## Design
### Approach: Gossip-Propagated LWW-Register CRDT
Each name binding is a **Last-Writer-Wins Register** — the most recent write (by timestamp) wins. This matches SWIM's eventual-consistency model and reuses the existing gossip piggyback mechanism.
**Why not Raft/consensus?**
- Overkill for name resolution. Names don't need linearizability — eventual consistency is fine.
- SWIM already solves dissemination. We piggyback registry updates on existing protocol messages for free.
- Consensus requires a stable quorum, which conflicts with the "nodes pop in and out" use case.
**Why not extend Kademlia?**
- Kademlia maps `ActorAddress -> NodeId`. Names are a different key space (`String -> ActorAddress`).
- Kademlia lookups are multi-hop (iterative). Registry lookups should be local (every node has a full replica).
- The registry is small (hundreds to low-thousands of names). Full replication is cheap.
### Types
```rust
// crates/distribution/src/registry.rs
/// A single name binding in the cluster registry.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegistryEntry {
/// Human-readable name (e.g. "worker-pool", "metrics-collector").
pub name: String,
/// The actor address this name resolves to.
pub actor_addr: ActorAddress,
/// The node that owns this binding.
pub node_id: NodeId,
/// Logical timestamp for LWW conflict resolution.
pub timestamp: u64,
/// Generation — incremented on re-registration of the same name.
pub generation: u64,
/// Tombstone — true means the name has been unregistered.
pub tombstone: bool,
}
/// Events emitted by the registry for subscribers.
#[derive(Debug, Clone)]
pub enum RegistryEvent {
/// A name was registered or updated.
Registered {
name: String,
actor_addr: ActorAddress,
node_id: NodeId,
},
/// A name was unregistered (tombstoned).
Unregistered {
name: String,
previous_addr: ActorAddress,
},
}
/// The local replica of the cluster-wide registry.
pub struct ClusterRegistry {
/// Current state: name -> latest entry.
entries: HashMap<String, RegistryEntry>,
/// Pending entries to propagate via gossip (not yet disseminated to all).
pending: VecDeque<RegistryEntry>,
/// Logical clock for this node.
clock: u64,
/// Recent events for subscribers.
events: VecDeque<RegistryEvent>,
/// Max events to buffer.
max_events: usize,
}
```
### CRDT Merge Rule
```rust
impl ClusterRegistry {
/// Merge a remote entry. Returns true if the local state changed.
pub fn merge(&mut self, remote: RegistryEntry) -> bool {
match self.entries.get(&remote.name) {
Some(local) => {
// LWW: higher timestamp wins.
// Tie-break: higher generation, then higher node_id (deterministic).
let dominated = remote.timestamp > local.timestamp
|| (remote.timestamp == local.timestamp
&& remote.generation > local.generation)
|| (remote.timestamp == local.timestamp
&& remote.generation == local.generation
&& remote.node_id.0 > local.node_id.0);
if dominated {
self.apply(remote);
true
} else {
false
}
}
None => {
self.apply(remote);
true
}
}
}
fn apply(&mut self, entry: RegistryEntry) {
let event = if entry.tombstone {
let prev = self.entries.get(&entry.name)
.map(|e| e.actor_addr);
RegistryEvent::Unregistered {
name: entry.name.clone(),
previous_addr: prev.unwrap_or_default(),
}
} else {
RegistryEvent::Registered {
name: entry.name.clone(),
actor_addr: entry.actor_addr,
node_id: entry.node_id,
}
};
self.events.push_back(event);
if self.events.len() > self.max_events {
self.events.pop_front();
}
self.entries.insert(entry.name.clone(), entry);
}
}
```
### API
On `DistributedNode`:
```rust
// crates/distribution/src/node.rs
impl DistributedNode {
/// Register a name -> actor binding on this node.
/// The binding is propagated to all cluster members via gossip.
pub fn register_name(&mut self, name: &str, actor_addr: ActorAddress) {
self.registry.clock += 1;
let entry = RegistryEntry {
name: name.to_string(),
actor_addr,
node_id: self.node_id(),
timestamp: self.registry.clock,
generation: self.registry.next_generation(name),
tombstone: false,
};
self.registry.merge(entry.clone());
self.registry.pending.push_back(entry);
}
/// Remove a name binding. Propagated as a tombstone.
pub fn unregister_name(&mut self, name: &str) {
self.registry.clock += 1;
let actor_addr = self.registry.entries.get(name)
.map(|e| e.actor_addr)
.unwrap_or_default();
let entry = RegistryEntry {
name: name.to_string(),
actor_addr,
node_id: self.node_id(),
timestamp: self.registry.clock,
generation: 0,
tombstone: true,
};
self.registry.merge(entry.clone());
self.registry.pending.push_back(entry);
}
/// Resolve a name to an actor address (local replica, eventually consistent).
pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
self.registry.entries.get(name)
.filter(|e| !e.tombstone)
.map(|e| (e.actor_addr, e.node_id))
}
/// Drain buffered registry events (for subscribers).
pub fn registry_events(&mut self) -> Vec<RegistryEvent> {
self.registry.events.drain(..).collect()
}
}
```
On `Ctx` (actor-level, requires distribution feature):
```rust
// src/actor.rs — requires ContextInner extensions
impl Ctx<'_> {
/// Register this actor under a name in the cluster registry.
pub fn register_as(&self, name: &str) {
self.inner.register_name(self.self_addr, name);
}
/// Resolve a name to an actor address.
pub fn resolve_name(&self, name: &str) -> Option<ActorAddress> {
self.inner.resolve_name(name)
}
}
```
The `ContextInner` trait gains two new methods:
```rust
pub trait ContextInner {
// ... existing methods ...
fn register_name(&self, addr: ActorAddress, name: &str) { /* default no-op */ }
fn resolve_name(&self, name: &str) -> Option<ActorAddress> { None }
}
```
Default implementations return `None` / no-op so that non-distributed runtimes don't break.
### Gossip Propagation
Registry entries are piggybacked on SWIM protocol messages, reusing the existing dissemination mechanism.
Currently, `crates/distribution/src/swim/dissemination.rs` encodes membership updates into the piggyback payload:
```
piggyback bytes = bincode(Vec<MembershipUpdate>)
```
Extended format:
```
piggyback bytes = bincode(PiggybackPayload {
membership: Vec<MembershipUpdate>,
registry: Vec<RegistryEntry>, // NEW
})
```
```rust
// crates/distribution/src/swim/dissemination.rs
#[derive(Serialize, Deserialize)]
struct PiggybackPayload {
membership: Vec<MembershipUpdate>,
registry: Vec<RegistryEntry>,
}
```
The dissemination buffer manages registry entries the same way as membership updates:
- Each entry has a dissemination count (how many times it's been piggybacked).
- After `log2(N) + 1` disseminations (where N = cluster size), the entry is retired.
- Piggyback space is shared: membership updates take priority, registry entries fill remaining space.
### Node Death Handling
When SWIM marks a node as `Dead`:
```rust
fn handle_membership_change(&mut self, node_id: NodeId, state: MemberState) {
if state == MemberState::Dead {
// ... existing cleanup ...
// NEW: tombstone all registry entries owned by the dead node
let to_tombstone: Vec<String> = self.registry.entries.iter()
.filter(|(_, e)| e.node_id == node_id && !e.tombstone)
.map(|(name, _)| name.clone())
.collect();
for name in to_tombstone {
self.registry.clock += 1;
let entry = RegistryEntry {
name: name.clone(),
tombstone: true,
timestamp: self.registry.clock,
// ... fill from existing entry ...
};
self.registry.merge(entry.clone());
self.registry.pending.push_back(entry);
}
}
}
```
### Interaction with Actor Watching
The registry and watching system compose naturally:
1. Actor A resolves name "service-X" → gets address B on Node 2.
2. Actor A calls `ctx.watch(B)`.
3. Node 2 dies. Actor A receives `ActorExited { addr: B, reason: NodeDown }`.
4. A supervisor re-spawns "service-X" on Node 3 → new address C.
5. The supervisor calls `register_name("service-X", C)`.
6. Gossip propagates the update.
7. Actor A (or anyone) calls `resolve_name("service-X")` → gets address C.
8. Actor A calls `ctx.watch(C)` to resume monitoring.
### Tombstone Garbage Collection
Tombstones accumulate over time. GC strategy:
- Tombstones older than `tombstone_ttl` (default: 1 hour of logical clock ticks) are eligible for removal.
- GC runs periodically (e.g., every 1000 ticks).
- A tombstone is only removed if it has been fully disseminated (dissemination count >= threshold).
### Files Modified
| File | Change |
|------|--------|
| `crates/distribution/src/registry.rs` | **New file**: `ClusterRegistry`, `RegistryEntry`, `RegistryEvent`, CRDT merge |
| `crates/distribution/src/lib.rs` | `pub mod registry;` |
| `crates/distribution/src/node.rs` | `register_name`, `unregister_name`, `resolve_name`, node death tombstoning |
| `crates/distribution/src/swim/dissemination.rs` | `PiggybackPayload` extended with registry entries |
| `src/actor.rs` | `register_name`/`resolve_name` on `ContextInner` (default no-op), `Ctx` wrappers |
### Tests
- **register_and_resolve**: register a name, resolve it, verify correct address
- **lww_conflict**: two nodes register same name concurrently, verify latest timestamp wins
- **tombstone_propagation**: register name, unregister, verify tombstone propagates and resolve returns None
- **node_death_tombstones**: 3-node cluster, register name on node B, kill node B, verify name is tombstoned on surviving nodes
- **re_registration**: register name, unregister, re-register with new address, verify resolution
- **gossip_convergence**: register name on node A, verify all nodes resolve it after gossip settles

View file

@ -0,0 +1,306 @@
# Node Capabilities — Hardware Detection & Placement Constraints
## Problem
Swactor targets heterogeneous clusters: some nodes have GPUs, others have large RAM, others are lightweight ARM devices. When spawning an actor (e.g., a model inference worker), the system needs to place it on a node with the right hardware. Today, placement is round-robin — no awareness of what each node can do.
## Design
### Separate Crate
`crates/capabilities/` is a **standalone crate** with no dependency on the swactor core runtime. It's a pure detection + constraint-matching library.
```toml
# crates/capabilities/Cargo.toml
[package]
name = "swactor-capabilities"
version = "0.1.0"
edition = "2024"
[features]
default = ["detect"]
detect = ["dep:sysinfo"]
gpu-nvidia = []
# gpu-vulkan = [] # future
[dependencies]
serde = { version = "1", features = ["derive"] }
sysinfo = { version = "0.33", optional = true }
```
### Types
```rust
// crates/capabilities/src/lib.rs
/// A capability value. Kept simple — three variants cover all practical needs.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum CapValue {
Bool(bool),
Int(i64),
Str(String),
}
/// All capabilities of a node.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct NodeCapabilities {
labels: BTreeMap<String, CapValue>,
}
impl NodeCapabilities {
pub fn new() -> Self { Self::default() }
/// Get a label value.
pub fn get(&self, key: &str) -> Option<&CapValue> {
self.labels.get(key)
}
/// Set a label.
pub fn set(&mut self, key: impl Into<String>, value: CapValue) {
self.labels.insert(key.into(), value);
}
/// Merge in additional labels (overwriting on conflict).
pub fn with_labels(mut self, extra: BTreeMap<String, CapValue>) -> Self {
self.labels.extend(extra);
self
}
/// Check if all constraints in a requirement are satisfied.
pub fn satisfies(&self, requirement: &PlacementRequirement) -> bool {
requirement.constraints.iter().all(|c| self.satisfies_one(c))
}
fn satisfies_one(&self, constraint: &PlacementConstraint) -> bool {
match constraint {
PlacementConstraint::Equals(key, expected) => {
self.labels.get(key.as_str()) == Some(expected)
}
PlacementConstraint::MinInt(key, min) => {
matches!(self.labels.get(key.as_str()), Some(CapValue::Int(v)) if *v >= *min)
}
PlacementConstraint::HasLabel(key) => {
self.labels.contains_key(key.as_str())
}
}
}
/// All labels as a reference.
pub fn labels(&self) -> &BTreeMap<String, CapValue> {
&self.labels
}
}
```
### Auto-Detection
```rust
impl NodeCapabilities {
/// Auto-detect system capabilities.
/// Always detects arch and os. Feature-gated backends detect more.
pub fn detect() -> Self {
let mut caps = Self::new();
// Always available (no feature gate)
caps.set("arch", CapValue::Str(std::env::consts::ARCH.to_string()));
caps.set("os", CapValue::Str(std::env::consts::OS.to_string()));
#[cfg(feature = "detect")]
{
Self::detect_sysinfo(&mut caps);
}
#[cfg(feature = "gpu-nvidia")]
{
Self::detect_nvidia(&mut caps);
}
caps
}
#[cfg(feature = "detect")]
fn detect_sysinfo(caps: &mut Self) {
use sysinfo::System;
let sys = System::new_all();
caps.set("cpu_count", CapValue::Int(sys.cpus().len() as i64));
caps.set("ram_mb", CapValue::Int((sys.total_memory() / (1024 * 1024)) as i64));
if let Ok(hostname) = hostname::get() {
if let Some(name) = hostname.to_str() {
caps.set("hostname", CapValue::Str(name.to_string()));
}
}
}
#[cfg(feature = "gpu-nvidia")]
fn detect_nvidia(caps: &mut Self) {
// Shell out to nvidia-smi for maximum compatibility.
// Parsing XML output is more robust than CSV for varying driver versions.
let output = std::process::Command::new("nvidia-smi")
.args(["--query-gpu=name,memory.total", "--format=csv,noheader,nounits"])
.output();
match output {
Ok(out) if out.status.success() => {
let stdout = String::from_utf8_lossy(&out.stdout);
let lines: Vec<&str> = stdout.trim().lines().collect();
caps.set("gpu_nvidia", CapValue::Bool(true));
caps.set("gpu_count", CapValue::Int(lines.len() as i64));
// First GPU's VRAM as representative
if let Some(line) = lines.first() {
let parts: Vec<&str> = line.split(", ").collect();
if let Some(name) = parts.first() {
caps.set("gpu_name", CapValue::Str(name.trim().to_string()));
}
if let Some(vram) = parts.get(1).and_then(|s| s.trim().parse::<i64>().ok()) {
caps.set("gpu_vram_mb", CapValue::Int(vram));
}
}
}
_ => {
caps.set("gpu_nvidia", CapValue::Bool(false));
}
}
}
}
```
### Placement Constraints
```rust
/// A single constraint on node capabilities.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PlacementConstraint {
/// Label must exist and equal the given value.
Equals(String, CapValue),
/// Label must exist and be >= the given integer value.
MinInt(String, i64),
/// Label must exist (any value).
HasLabel(String),
}
/// A full placement requirement. All constraints must be satisfied (AND).
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PlacementRequirement {
pub constraints: Vec<PlacementConstraint>,
}
impl PlacementRequirement {
pub fn new() -> Self { Self::default() }
/// Builder: require a label equals a value.
pub fn equals(mut self, key: impl Into<String>, value: CapValue) -> Self {
self.constraints.push(PlacementConstraint::Equals(key.into(), value));
self
}
/// Builder: require an integer label >= min.
pub fn min_int(mut self, key: impl Into<String>, min: i64) -> Self {
self.constraints.push(PlacementConstraint::MinInt(key.into(), min));
self
}
/// Builder: require a label exists.
pub fn has(mut self, key: impl Into<String>) -> Self {
self.constraints.push(PlacementConstraint::HasLabel(key.into()));
self
}
/// Check if empty (no constraints — any node is acceptable).
pub fn is_empty(&self) -> bool {
self.constraints.is_empty()
}
}
```
### Integration with Distribution
**NodeRecord extension** (`crates/distribution/src/types.rs`):
```rust
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeRecord {
pub node_id: NodeId,
pub addr: SocketAddr,
pub state: MemberState,
pub incarnation: u64,
// NEW (optional — backwards compatible):
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capabilities: Option<swactor_capabilities::NodeCapabilities>,
}
```
**Capabilities flow**:
1. On startup, the node detects capabilities: `NodeCapabilities::detect().with_labels(operator_labels)`.
2. Capabilities are included in the node's own `NodeRecord`.
3. When a node joins (via `JoinResponse`), it receives other nodes' capabilities.
4. Capabilities are piggybacked on SWIM protocol messages (membership updates already carry `NodeRecord`).
**Cluster-level placement** (new function in distribution):
```rust
// crates/distribution/src/node.rs
impl DistributedNode {
/// Find nodes that satisfy a placement requirement.
/// Returns matching nodes sorted by preference (e.g., least loaded first).
pub fn find_suitable_nodes(
&self,
requirement: &PlacementRequirement,
) -> Vec<NodeRecord> {
self.members()
.into_iter()
.filter(|node| {
node.capabilities.as_ref()
.map(|caps| caps.satisfies(requirement))
.unwrap_or(requirement.is_empty())
})
.collect()
}
}
```
### Example Usage
```rust
// Operator starts a node with custom labels:
let caps = NodeCapabilities::detect()
.with_labels(btreemap! {
"role".into() => CapValue::Str("inference".into()),
"region".into() => CapValue::Str("us-east".into()),
});
// An actor specifies placement requirements:
let requirement = PlacementRequirement::new()
.has("gpu_nvidia")
.min_int("gpu_vram_mb", 8000)
.equals("region", CapValue::Str("us-east".into()));
// Supervisor finds suitable nodes:
let nodes = dist_node.find_suitable_nodes(&requirement);
```
### Files Modified
| File | Change |
|------|--------|
| `crates/capabilities/` | **New crate** |
| `crates/capabilities/Cargo.toml` | Package definition, feature flags |
| `crates/capabilities/src/lib.rs` | `NodeCapabilities`, `CapValue`, `PlacementConstraint`, `PlacementRequirement`, detection |
| `crates/distribution/Cargo.toml` | Optional dependency on `swactor-capabilities` |
| `crates/distribution/src/types.rs` | Optional `capabilities` field on `NodeRecord` |
| `crates/distribution/src/node.rs` | `find_suitable_nodes()`, capabilities in join flow |
| `Cargo.toml` | Add `crates/capabilities` to workspace members |
### Tests
- **detect_basics**: `NodeCapabilities::detect()` always has `arch` and `os` labels
- **satisfies_equals**: constraint matches/doesn't match
- **satisfies_min_int**: integer comparison works correctly
- **satisfies_has_label**: existence check works
- **empty_requirement**: matches any node
- **combined_constraints**: multiple constraints all must pass (AND)
- **custom_labels**: operator labels merge correctly, override detection
- **find_suitable_nodes**: integration test with mock node records and varying capabilities

View file

@ -0,0 +1,398 @@
# Command Interface — Frontend-Agnostic Dispatch
## Problem
The runtime has an investigate/REPL protocol (`crates/runtime-dashboard/src/investigate.rs`) that accepts text commands and returns JSON. It works, but it's hardcoded to stdin/stdout and tightly coupled to the dashboard crate. We need the same commands accessible from:
- Terminal CLI (stdin/stdout)
- TUI (the existing ratatui dashboard)
- REST API (the existing HTTP server)
- Future: WebSocket, remote CLI, programmatic SDK
And we need write commands (spawn, stop, drain) — not just read-only inspection.
## Design
### Architecture
```
┌──────────────────────────────────────────────────────────┐
│ Frontends │
│ │
│ ┌──────────┐ ┌─────────┐ ┌──────────┐ ┌──────────┐ │
│ │CLI / REPL│ │ TUI │ │REST /api/│ │ Future │ │
│ │(stdin/ │ │(ratatui │ │cmd?name= │ │(websocket│ │
│ │ stdout) │ │ events) │ │&arg=val │ │ etc.) │ │
│ └────┬─────┘ └────┬────┘ └────┬─────┘ └────┬─────┘ │
│ │ │ │ │ │
│ └─────────────┴─────┬──────┴──────────────┘ │
└───────────────────────────┼──────────────────────────────┘
│
┌────────▼────────┐
│ CommandRouter │
│ │
│ name → handler │
│ dispatch() │
└────────┬────────┘
│
┌──────────────┼──────────────┐
│ │ │
┌──────▼──────┐ ┌────▼────┐ ┌───────▼───────┐
│ Built-in │ │Built-in │ │ Custom │
│ Read Cmds │ │Write │ │ (actor- │
│ (overview, │ │Cmds │ │ registered) │
│ workers, │ │(spawn, │ │ │
│ actors, │ │ stop, │ │ │
│ hot, ...) │ │ drain) │ │ │
└─────────────┘ └─────────┘ └────────────────┘
```
### Core Types
```rust
// crates/command/src/lib.rs
use std::collections::HashMap;
use std::sync::Arc;
/// A command request.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandRequest {
/// Command name (e.g., "overview", "spawn", "actors").
pub command: String,
/// Named arguments. Values are JSON for flexibility.
pub args: HashMap<String, serde_json::Value>,
}
/// A command response. Always JSON-serializable.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CommandResponse {
pub ok: bool,
pub command: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub data: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
impl CommandResponse {
pub fn ok(command: &str, data: impl Serialize) -> Self {
Self {
ok: true,
command: command.to_string(),
data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)),
error: None,
}
}
pub fn err(command: &str, msg: impl Into<String>) -> Self {
Self {
ok: false,
command: command.to_string(),
data: None,
error: Some(msg.into()),
}
}
/// Serialize to a single JSON line (for REPL protocol).
pub fn to_json_line(&self) -> String {
serde_json::to_string(self).unwrap_or_else(|e| {
format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#)
})
}
}
```
### CommandHandler Trait
```rust
/// Metadata about a command, used for help text and validation.
pub struct CommandMeta {
pub name: &'static str,
pub description: &'static str,
pub usage: &'static str,
/// Whether this command mutates state (spawn, stop, etc.)
pub is_write: bool,
}
/// A command handler. Implementations are stateless — all state
/// comes through CommandContext.
pub trait CommandHandler: Send + Sync {
fn meta(&self) -> CommandMeta;
fn handle(&self, args: &HashMap<String, serde_json::Value>, ctx: &CommandContext) -> CommandResponse;
}
```
### CommandContext
```rust
/// Context available to command handlers.
///
/// Contains references to runtime subsystems. Optional fields allow
/// commands to work in both standalone and distributed configurations.
pub struct CommandContext {
pub runtime: Arc<swactor::runtime::Runtime>,
pub stats_collector: Option<Arc<crate::StatsCollector>>,
// Distribution (only present when running distributed)
pub dist_node: Option<Arc<std::sync::Mutex<distribution::node::DistributedNode>>>,
// Extensibility: arbitrary typed data that custom commands can access
extensions: HashMap<std::any::TypeId, Box<dyn std::any::Any + Send + Sync>>,
}
impl CommandContext {
/// Retrieve a typed extension.
pub fn get_ext<T: 'static + Send + Sync>(&self) -> Option<&T> {
self.extensions.get(&std::any::TypeId::of::<T>())
.and_then(|b| b.downcast_ref())
}
/// Add a typed extension.
pub fn set_ext<T: 'static + Send + Sync>(&mut self, val: T) {
self.extensions.insert(std::any::TypeId::of::<T>(), Box::new(val));
}
}
```
### CommandRouter
```rust
/// Central command dispatch.
pub struct CommandRouter {
handlers: HashMap<String, Box<dyn CommandHandler>>,
}
impl CommandRouter {
pub fn new() -> Self {
Self { handlers: HashMap::new() }
}
/// Register all built-in commands.
pub fn with_builtins(mut self) -> Self {
self.register(Box::new(builtins::HelpCommand));
self.register(Box::new(builtins::OverviewCommand));
self.register(Box::new(builtins::WorkersCommand));
self.register(Box::new(builtins::WorkerCommand));
self.register(Box::new(builtins::ActorsCommand));
self.register(Box::new(builtins::ActorCommand));
self.register(Box::new(builtins::HotCommand));
self.register(Box::new(builtins::PhasesCommand));
self.register(Box::new(builtins::DiffCommand));
// Write commands
self.register(Box::new(builtins::SpawnCommand));
self.register(Box::new(builtins::StopCommand));
self.register(Box::new(builtins::ShutdownCommand));
// Distribution-aware commands (no-op if dist_node is None)
self.register(Box::new(builtins::NodesCommand));
self.register(Box::new(builtins::RegistryCommand));
self.register(Box::new(builtins::ResolveCommand));
self.register(Box::new(builtins::DrainCommand));
self
}
/// Register a custom command handler.
pub fn register(&mut self, handler: Box<dyn CommandHandler>) {
let name = handler.meta().name.to_string();
self.handlers.insert(name, handler);
}
/// Dispatch a command request.
pub fn dispatch(&self, req: &CommandRequest, ctx: &CommandContext) -> CommandResponse {
match self.handlers.get(&req.command) {
Some(handler) => handler.handle(&req.args, ctx),
None => CommandResponse::err(
&req.command,
format!("unknown command `{}` — try `help`", req.command),
),
}
}
/// List all registered commands (for help text).
pub fn commands(&self) -> Vec<&CommandMeta> {
// sorted by name for stable output
let mut metas: Vec<_> = self.handlers.values()
.map(|h| h.meta())
.collect();
metas.sort_by_key(|m| m.name);
metas
}
}
```
### Built-in Commands (MVP)
**Read commands** (extracted from existing `investigate.rs`):
| Command | Args | Description | Source |
|---------|------|-------------|--------|
| `help` | — | List all commands | `cmd_help()` |
| `overview` | — | Runtime summary | `cmd_overview()` |
| `workers` | — | Per-worker stats | `cmd_workers()` |
| `worker` | `id: int` | Single worker detail | `cmd_worker()` |
| `actors` | `sort`, `limit`, `worker` | List actors | `cmd_actors()` |
| `actor` | `prefix: str` | Find by address prefix | `cmd_actor()` |
| `hot` | `n: int` | Top N by mailbox depth | `cmd_hot()` |
| `phases` | `worker: int?` | Tick phase breakdown | `cmd_phases()` |
| `diff` | `seconds: float` | Snapshot delta | `cmd_diff()` |
| `nodes` | — | Cluster member list | **new** |
| `registry` | — | All registered names | **new** |
| `resolve` | `name: str` | Look up a name | **new** |
**Write commands** (new):
| Command | Args | Description |
|---------|------|-------------|
| `stop` | `prefix` or `name` | Stop an actor (poison + cleanup) |
| `drain` | `node: str?` | Stop accepting new actors on a node, let existing drain |
| `shutdown` | `node: str?` | Graceful shutdown (drain + stop all) |
| `spawn` | `factory`, `node?`, `constraints?` | Spawn from a registered factory |
`spawn` requires a **factory registry** — actors register factory functions that can be invoked by name:
```rust
pub trait ActorFactory: Send + Sync {
fn name(&self) -> &str;
fn spawn(&self, runtime: &Runtime, args: &HashMap<String, serde_json::Value>)
-> Result<ActorAddress, String>;
}
```
### Migration from investigate.rs
The existing `crates/runtime-dashboard/src/investigate.rs` has 9 command functions. Migration strategy:
1. Create `crates/command/src/builtins/` with one file per command (or grouped by category).
2. Each `cmd_*` function becomes a `CommandHandler` impl. The logic is identical — just restructured.
3. `dispatch_repl` becomes `CommandRouter::dispatch` with a text-to-`CommandRequest` parser.
4. `dispatch_command` (HTTP) becomes `CommandRouter::dispatch` with query-param-to-`CommandRequest` parser.
5. `run_investigate` remains in the dashboard crate as a thin loop over `CommandRouter`.
Example extraction:
```rust
// crates/command/src/builtins/overview.rs
pub struct OverviewCommand;
impl CommandHandler for OverviewCommand {
fn meta(&self) -> CommandMeta {
CommandMeta {
name: "overview",
description: "Summary: worker count, actor count, total messages, panics",
usage: "overview",
is_write: false,
}
}
fn handle(&self, _args: &HashMap<String, serde_json::Value>, ctx: &CommandContext) -> CommandResponse {
let stats = ctx.enriched_stats();
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
// ... same logic as existing cmd_overview ...
CommandResponse::ok("overview", serde_json::json!({
"workers": stats.num_workers,
"actors": stats.actor_details.len(),
"total_messages_processed": total_msgs,
// ...
}))
}
}
```
### Frontend Adapters
Each frontend is a thin adapter that converts its input format into `CommandRequest` and `CommandResponse` back to its output format.
**REPL adapter** (stdin/stdout):
```rust
// crates/command/src/adapters/repl.rs
pub fn parse_line(line: &str) -> CommandRequest {
let parts: Vec<&str> = line.split_whitespace().collect();
let command = parts.first().unwrap_or(&"help").to_string();
let args = parse_positional_and_flags(&parts[1..]);
CommandRequest { command, args }
}
```
**REST adapter** (HTTP query params):
```rust
// crates/command/src/adapters/rest.rs
pub fn from_query_params(params: &HashMap<String, String>) -> CommandRequest {
let command = params.get("cmd").cloned().unwrap_or_else(|| "help".into());
let args: HashMap<String, serde_json::Value> = params.iter()
.filter(|(k, _)| *k != "cmd")
.map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone())))
.collect();
CommandRequest { command, args }
}
```
**TUI adapter**: TUI input field text → `parse_line()` → `dispatch()` → render response in panel.
### Custom Commands (actor-registered)
Actors can register command handlers at runtime through the `Ctx`:
```rust
impl Ctx<'_> {
/// Register a command that routes to this actor.
/// When the command is invoked, a CommandInvocation message
/// is sent to this actor's mailbox.
pub fn register_command(&self, name: &str, description: &str) {
self.inner.register_command(self.self_addr, name, description);
}
}
```
When a custom command is dispatched:
1. Router finds it's actor-registered.
2. Sends `CommandInvocation { command, args, reply_addr }` to the actor's mailbox.
3. The actor processes it and sends `CommandResult { data }` back to `reply_addr`.
4. Router waits on a one-shot inbox (with timeout, e.g. 5 seconds).
5. Returns the response.
```rust
/// Sent to an actor when its registered command is invoked.
#[derive(Debug, Clone)]
pub struct CommandInvocation {
pub command: String,
pub args: HashMap<String, serde_json::Value>,
pub reply_addr: ActorAddress,
}
/// Sent back by the actor with the command result.
#[derive(Debug, Clone)]
pub struct CommandResult {
pub data: serde_json::Value,
}
```
This mechanism allows any actor to expose operational endpoints without modifying the command crate.
### Files Modified
| File | Change |
|------|--------|
| `crates/command/` | **New crate** |
| `crates/command/Cargo.toml` | Dependencies: swactor, serde, serde_json |
| `crates/command/src/lib.rs` | `CommandRouter`, `CommandHandler`, `CommandRequest`/`Response`, `CommandContext` |
| `crates/command/src/builtins/` | Built-in command handlers (mod.rs + per-command files) |
| `crates/command/src/adapters/` | REPL and REST input parsers |
| `crates/runtime-dashboard/src/investigate.rs` | Refactored: thin REPL loop over CommandRouter |
| `crates/runtime-dashboard/src/server.rs` | REST endpoints use CommandRouter |
| `crates/runtime-dashboard/Cargo.toml` | Depends on `crates/command` |
| `Cargo.toml` | Add `crates/command` to workspace |
### Tests
- **dispatch_known_command**: `overview` returns `ok: true` with expected fields
- **dispatch_unknown_command**: returns `ok: false` with helpful error
- **help_lists_all**: `help` response includes all registered command names
- **parse_repl_line**: `"actors --sort mailbox --limit 5"` → correct `CommandRequest`
- **parse_query_params**: `{cmd: "actor", prefix: "a1b2"}` → correct `CommandRequest`
- **custom_command_dispatch**: register actor command, invoke, verify response
- **custom_command_timeout**: registered actor doesn't respond, verify timeout error
- **write_command_stop**: stop an actor via command, verify it's poisoned

View file

@ -0,0 +1,263 @@
# Supervision — User-Space Self-Healing
## Problem
When nodes churn (spot instances preempted, hardware rebooted, network partitions), actors on those nodes are lost. Something needs to detect the loss and re-spawn the actors on surviving nodes. This is the "self-healing" property of a distributed OS.
## Design Principle: Supervision Is User-Space
Supervision is **not** a runtime primitive. It is a pattern built from:
- **Actor Watching** (01) — detect death
- **Cluster Registry** (02) — re-register under the same name
- **Node Capabilities** (03) — find a suitable replacement node
The runtime provides the low-level mechanisms. Supervision is a library actor that composes them. This keeps the kernel minimal and lets users customize supervision policy without forking the runtime.
## Supervisor Actor
```rust
/// A supervised child definition.
struct SupervisedChild {
/// Human-readable name (registered in cluster registry).
name: String,
/// Factory function to create the actor.
factory: Box<dyn ActorFactory>,
/// Placement constraints for the child.
constraints: PlacementRequirement,
/// Current address (None if not yet spawned or dead).
current_addr: Option<ActorAddress>,
/// Number of restarts so far.
restart_count: u32,
/// Maximum restarts before giving up (0 = unlimited).
max_restarts: u32,
/// Backoff state for restart delays.
last_restart: Option<Instant>,
}
/// Restart strategy for a supervisor.
#[derive(Debug, Clone)]
enum RestartStrategy {
/// Restart only the failed child.
OneForOne,
/// If any child fails, restart all children.
AllForOne,
/// Don't restart — just notify (for monitoring supervisors).
Notify,
}
/// The supervisor actor.
struct Supervisor {
children: Vec<SupervisedChild>,
strategy: RestartStrategy,
}
```
## Message Protocol
```rust
/// Messages the supervisor handles.
enum SupervisorMsg {
/// A watched child died.
Exited(ActorExited),
/// External request to add a child.
AddChild {
name: String,
factory: Box<dyn ActorFactory>,
constraints: PlacementRequirement,
},
/// External request to remove a child.
RemoveChild { name: String },
/// Query: what children are running?
Status { reply_to: ActorAddress },
}
```
## Lifecycle
### Startup
```rust
impl Supervisor {
fn start(&mut self, ctx: &Ctx) {
for child in &mut self.children {
match self.spawn_child(ctx, child) {
Ok(addr) => {
child.current_addr = Some(addr);
ctx.watch(addr);
ctx.register_as(&child.name); // or register the child
}
Err(e) => {
eprintln!("supervisor: failed to spawn {}: {e}", child.name);
}
}
}
}
}
```
### Death Handling
```rust
impl ActorInterface for Supervisor {
type Incoming = SupervisorMsg;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) {
match msg {
SupervisorMsg::Exited(exited) => {
match self.strategy {
RestartStrategy::OneForOne => {
self.restart_one(ctx, &exited);
}
RestartStrategy::AllForOne => {
self.restart_all(ctx);
}
RestartStrategy::Notify => {
// Just log — don't restart
}
}
}
// ... other messages ...
}
}
}
```
### Restart Flow (OneForOne)
```
1. Receive ActorExited { addr: X, reason: NodeDown }
2. Find child with current_addr == X → child "worker-3"
3. Check restart_count < max_restarts
4. Query capabilities: find_suitable_nodes(child.constraints)
5. Pick best node (least loaded, or local if possible)
6. Spawn child on selected node via factory
7. Watch new address
8. Register child.name → new address in cluster registry
9. Update child.current_addr
10. Increment child.restart_count
```
```rust
impl Supervisor {
fn restart_one(&mut self, ctx: &Ctx, exited: &ActorExited) {
let child = match self.children.iter_mut()
.find(|c| c.current_addr == Some(exited.addr))
{
Some(c) => c,
None => return, // not our child
};
child.current_addr = None;
if child.max_restarts > 0 && child.restart_count >= child.max_restarts {
eprintln!(
"supervisor: child {} exceeded max restarts ({}), giving up",
child.name, child.max_restarts,
);
return;
}
// Spawn replacement
match self.spawn_child(ctx, child) {
Ok(addr) => {
child.current_addr = Some(addr);
child.restart_count += 1;
ctx.watch(addr);
// Re-register name → new address
// (done via the registry, which gossips to all nodes)
}
Err(e) => {
eprintln!("supervisor: failed to restart {}: {e}", child.name);
}
}
}
}
```
### Restart Flow (AllForOne)
When any child dies:
1. Stop all other children (send stop signal).
2. Wait for all `ActorExited` notifications.
3. Restart all children in order.
This is useful for interdependent actor groups where partial restart doesn't make sense.
## Capability-Aware Placement
The supervisor uses `find_suitable_nodes()` from the distribution crate:
```rust
fn spawn_child(&self, ctx: &Ctx, child: &SupervisedChild)
-> Result<ActorAddress, String>
{
// If constraints are empty, spawn locally
if child.constraints.is_empty() {
return child.factory.spawn(ctx);
}
// Find suitable remote nodes
let nodes = dist_node.find_suitable_nodes(&child.constraints);
if nodes.is_empty() {
return Err("no nodes satisfy placement constraints".into());
}
// Pick the least-loaded suitable node
let target_node = &nodes[0]; // TODO: sort by load
// Spawn remotely (requires remote spawn protocol — future work)
// For now: if current node satisfies, spawn locally
// Otherwise: send spawn request to target node
todo!("remote spawn")
}
```
**Note**: Remote spawn (telling another node to create an actor) is not yet part of the runtime. The supervisor design accounts for it, but the initial implementation will only support local spawn + re-registration.
## Relationship to Existing `spawn_restartable`
The core runtime already has `spawn_restartable` with `factory` and `max_restarts`. This is a **local-only** recovery mechanism — when an actor panics, the same worker restarts it.
The supervisor pattern extends this to **cluster-wide** recovery:
| Feature | `spawn_restartable` | Supervisor |
|---|---|---|
| Scope | Single worker | Cluster-wide |
| Trigger | Panic | Panic, stop, or node death |
| Placement | Same worker | Capability-aware, any node |
| Naming | No | Yes (cluster registry) |
| Strategy | Always restart | OneForOne, AllForOne, Notify |
| Implementation | Runtime internal | User-space actor |
They complement each other: `spawn_restartable` handles fast local recovery (no network round-trip); the supervisor handles node-level failures.
## Future Extensions
- **Restart backoff**: exponential backoff between restarts to avoid thrashing.
- **Health checks**: periodic health probes (not just death detection).
- **Cascading supervisors**: supervisor trees (supervisor watches sub-supervisor).
- **Declarative spec**: TOML/YAML file defining supervision topology, loaded at startup.
- **Migration (not restart)**: move a running actor's state to another node (requires persistence, out of scope).
## Files
This is a library actor, not a runtime change. Implementation lives in:
| File | Content |
|------|---------|
| `crates/supervision/src/lib.rs` | `Supervisor`, `SupervisedChild`, `RestartStrategy` |
| `crates/supervision/src/factory.rs` | `ActorFactory` trait, factory registry |
| `crates/supervision/Cargo.toml` | Depends on `swactor`, `swactor-capabilities`, `distribution` |
Or, if the scope doesn't warrant a separate crate, it can live in `src/supervision.rs` behind a feature flag.
## Tests
- **one_for_one_restart**: supervisor with 3 children, kill one, verify only that one restarts
- **all_for_one_restart**: supervisor with 3 children, kill one, verify all restart
- **max_restarts_exceeded**: child dies repeatedly, verify supervisor gives up after max
- **name_re_registration**: child dies and restarts, verify name resolves to new address
- **capability_placement**: child with GPU constraint, verify spawned on GPU node (or error if none available)
- **supervisor_itself_dies**: verify children are stopped (or orphaned — design decision)

153
docs/os-design/README.md Normal file
View file

@ -0,0 +1,153 @@
# Swactor Distributed OS — Design Overview
## Vision
Swactor is evolving from a local actor runtime into a **distributed operating system** for running long-lived daemons across heterogeneous, churning machines.
**Primary use case**: Wire together personal hardware today; add on-demand spot compute (vast.ai, etc.) tomorrow. Machines pop in and out of the network. The system self-heals.
**Design principles**:
- **Churn is the norm**, not the exception. Every subsystem assumes nodes can disappear at any time.
- **General and flexible**. Minimal assumptions about what a node looks like — feature-gate hardware-specific code.
- **Layered**. The core runtime stays minimal. OS features are opt-in crates. Supervision is user-space, not kernel.
- **Frontend-agnostic**. Operational interfaces (commands, inspection) work identically from CLI, TUI, REST, or future transports.
## Current Capabilities
| OS Concept | What Swactor Has Today |
|---|---|
| Processes | Actor spawn/stop, lifecycle hooks, restartable with factory + max_restarts |
| Scheduling | Worker threads, fairness budget (64 msgs/tick), load-aware placement |
| IPC | Typed send, request/reply, per-actor VecDeque mailboxes |
| Naming | String-keyed `AddressMap` — node-local only |
| Fault tolerance | `catch_unwind` for panics, factory restart, dead actor cleanup |
| Backpressure | Per-actor mailbox capacity, DropNewest/DropOldest overflow |
| Networking | SWIM membership (Lifeguard extensions), TCP transport, bincode wire protocol |
| Directory | Kademlia DHT for `ActorAddress -> NodeId` resolution |
| Monitoring | WorkerStats/RuntimeStats, TUI dashboard, REST `/api/*`, investigate REPL |
| Distribution | Multi-node cluster, 5-node Docker test suite |
## What's Missing (This Design)
| Feature | Document | Priority |
|---|---|---|
| Actor Watching | [01-actor-watching.md](./01-actor-watching.md) | Foundation for everything |
| Cluster Registry | [02-cluster-registry.md](./02-cluster-registry.md) | Actors find each other across nodes |
| Node Capabilities | [03-node-capabilities.md](./03-node-capabilities.md) | Heterogeneous placement |
| Command Interface | [04-command-interface.md](./04-command-interface.md) | Operational control |
| Supervision | [05-supervision.md](./05-supervision.md) | Self-healing (user-space) |
## Crate Structure (After This Work)
```
swactor/ # core runtime — no network deps
src/
actor.rs # +watch/unwatch on ContextInner, ExitReason, ActorExited
worker.rs # +WatchRegistry, death notification phase
delivery.rs # (unchanged)
runtime.rs # (unchanged)
...
crates/
capabilities/ # NEW — hardware detection + placement constraints
src/lib.rs # NodeCapabilities, CapValue, PlacementConstraint
command/ # NEW — frontend-agnostic command dispatch
src/lib.rs # CommandRouter, CommandHandler, CommandRequest/Response
src/builtins/ # Built-in command handlers
distribution/ # SWIM + Kademlia + cluster registry
src/
registry.rs # NEW — ClusterRegistry, gossip-propagated naming
node.rs # +register_name, resolve_name, capabilities
messages.rs # +WatchRequest, UnwatchRequest, ActorExitedNotify
...
runtime-dashboard/ # TUI + REST — refactored to use command crate
python/ # PyO3 bindings
simulation/ # Network simulation
wasm/ # WASM bindings
```
## Dependency Graph
```
┌──────────────┐
│ capabilities │ (standalone — only serde + sysinfo)
└──────┬───────┘
│ optional
┌─────────┐ ┌──────▼────────┐
│ swactor │◄─────│ distribution │
│ (core) │ │ +registry │
└────┬────┘ └──────┬────────┘
│ │
│ ┌──────▼────────┐
└──────────►│ command │
└──────┬────────┘
│
┌──────▼────────────┐
│ runtime-dashboard │
│ (TUI + REST) │
└───────────────────┘
```
Key constraints:
- `capabilities` has **zero** dependency on `swactor` — it's a standalone detection library.
- `command` depends on `swactor` (needs `Runtime`, stats types) but NOT on `distribution`.
- `distribution` optionally depends on `capabilities` (for `NodeRecord` labels).
- `runtime-dashboard` depends on both `command` and optionally `distribution`.
## Feature Flags
### Core `swactor`
| Flag | Purpose |
|---|---|
| `getrandom` (default) | Cryptographic RNG for actor addresses |
| `serde` | Serialization for types |
| `tracing` | Structured logging |
| `transport` | Transport-agnostic remote messaging |
| `watching` (new) | Watch API on ContextInner/Ctx |
### `swactor-capabilities`
| Flag | Purpose |
|---|---|
| `detect` (default) | Auto-detect CPU, RAM, hostname via sysinfo |
| `gpu-nvidia` | Detect NVIDIA GPU via nvidia-smi |
| `gpu-vulkan` | Detect GPU via Vulkan API |
### `distribution`
| Flag | Purpose |
|---|---|
| `registry` (new, default) | Cluster-wide gossip-propagated naming |
| `capabilities` (new) | NodeCapabilities on NodeRecord |
## Implementation Order
Each feature is one PR, in dependency order:
```
PR 1: Actor Watching (local only)
└──► PR 2: Command Interface
└──► PR 3: Cluster Registry
└──► PR 4: Node Capabilities
└──► PR 5: Remote Watching (cross-node)
└──► PR 6: Supervisor library
```
**PR 1 — Actor Watching (local)**
Adds `WatchRegistry`, `ActorExited`, `ExitReason` to core runtime. Testable without any distribution. Foundation for everything else.
**PR 2 — Command Interface**
Extracts investigate.rs into `crates/command/`. Adds write commands (spawn, stop, drain). Immediately useful for operations.
**PR 3 — Cluster Registry**
Gossip-propagated naming in `crates/distribution/src/registry.rs`. Actors can find each other by name across nodes.
**PR 4 — Node Capabilities**
`crates/capabilities/` with auto-detection and placement constraints. Integrates with distribution for capability-aware placement.
**PR 5 — Remote Watching**
Wire protocol for cross-node watches. SWIM Dead triggers `ActorExited { reason: NodeDown }` for all actors on that node.
**PR 6 — Supervisor Library**
User-space supervisor pattern. Composes watching + registry + capabilities to auto-respawn actors after churn.

115
docs/wasm-actor.md Normal file
View file

@ -0,0 +1,115 @@
# Wasm Actor
The `swactor-wasm-actor` crate runs WebAssembly guest code inside a swactor
actor. The Wasm instance is sandboxed by [wasmtime](https://wasmtime.dev/).
## Architecture
```
┌─ Runtime ──────────────────────────────────────────────────────────────┐
│ │
│ ┌─ WasmActor ──────────────────────────────────────────────────────┐ │
│ │ │ │
│ │ Store<HostState> -- wasmtime store with outbox │ │
│ │ Memory -- guest linear memory │ │
│ │ alloc: TypedFunc -- guest allocator │ │
│ │ handle: TypedFunc -- guest message handler │ │
│ │ │ │
│ │ impl ActorInterface for WasmActor │ │
│ │ Incoming = ByteMessage │ │
│ │ Response = () │ │
│ │ │ │
│ └──────────────────────────────────────────────────────────────────┘ │
│ │
│ ┌─ Native Actors ─────────────────────────────────────────────────┐ │
│ │ (can exchange ByteMessage with WasmActors normally) │ │
│ └─────────────────────────────────────────────────────────────────┘ │
│ │
└────────────────────────────────────────────────────────────────────────┘
```
## Message Flow
```
Host Guest (Wasm)
──── ────────────
ByteMessage arrives
│
├─1─ call alloc(len) ──────────► bump-allocate, return ptr
│
├─2─ write bytes at ptr ───────► (memory updated)
│
├─3─ call handle(ptr, len) ────► process message
│ │
│ ◄── swactor.send() ────────────┤ (0..N times)
│ (buffered in HostState.outbox) │
│ │
├─4─ drain outbox ◄────────────── handle returns
│
v
ctx.send(dest, ByteMessage) for each outbox entry
```
## Guest Contract
Guests are standalone `wasm32-unknown-unknown` modules. They export three
symbols and may import one:
| Direction | Module | Symbol | Signature |
|-----------|--------|--------|-----------|
| **export** | — | `memory` | linear memory |
| **export** | — | `alloc` | `(i32) -> i32` |
| **export** | — | `handle` | `(i32, i32) -> ()` |
| **import** | `swactor` | `send` | `(i32, i32, i32) -> ()` |
The `send` import takes `(dest_ptr, payload_ptr, payload_len)` where
`dest_ptr` points to a 32-byte `ActorAddress` in guest memory.
## Usage
```rust
use swactor::runtime::{Runtime, RuntimeConfig};
use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder};
// Create a shared engine (once)
let engine = SharedEngine::new().unwrap();
// Build an actor from .wasm bytes
let wasm_bytes = std::fs::read("my_guest.wasm").unwrap();
let actor = WasmActorBuilder::new(engine, wasm_bytes)
.build()
.unwrap();
// Use it like any other actor
let rt = Runtime::new(RuntimeConfig::default());
let addr = rt.spawn(actor).unwrap();
rt.send_to(addr, ByteMessage(b"hello".to_vec())).unwrap();
rt.tick();
```
## Sandboxing
The `SharedEngine` disables all optional Wasm proposals:
- Threads — disabled
- SIMD / relaxed SIMD — disabled
- Reference types — disabled
- Multi-value — disabled
- Bulk memory — **enabled** (required by most Rust/LLVM toolchains)
No WASI imports are linked. Guests have no access to the filesystem, network,
clock, or random number generator. The only host function available is
`swactor.send`.
## Where Things Live
| File | Purpose |
|------|---------|
| `crates/wasm-actor/src/lib.rs` | `ByteMessage` + re-exports |
| `crates/wasm-actor/src/engine.rs` | `SharedEngine` — sandboxed wasmtime config |
| `crates/wasm-actor/src/builder.rs` | `WasmActorBuilder` — compile, link, instantiate |
| `crates/wasm-actor/src/actor.rs` | `WasmActor` — `ActorInterface` impl |
| `crates/wasm-actor/src/error.rs` | `WasmActorError` |
| `crates/wasm-actor/tests/guests/` | Three test guest crates (echo, double, silent) |
| `crates/wasm-actor/tests/wasm_actor.rs` | 7 integration tests |

View file

@ -2,6 +2,31 @@ use std::any::Any;
use crate::Error; use crate::Error;
/// Why an actor exited.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ExitReason {
/// Actor was explicitly stopped or removed from the pool.
Stopped,
/// Actor panicked during message handling.
Panicked,
/// The node hosting the actor left the cluster (SWIM Dead).
NodeDown,
}
/// Delivered to watchers when a watched actor exits.
///
/// Implements `Message` (Clone + Send + Sync + 'static) so it can be
/// delivered through normal mailbox channels.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ActorExited {
/// The address of the actor that died.
pub addr: ActorAddress,
/// Why it exited.
pub reason: ExitReason,
}
/// The primary trait defining data that can be passed to and from actor processes /// The primary trait defining data that can be passed to and from actor processes
pub trait Message: 'static + Sized + Clone + Send + Sync {} pub trait Message: 'static + Sized + Clone + Send + Sync {}
impl<T: 'static + Sized + Clone + Send + Sync> Message for T {} impl<T: 'static + Sized + Clone + Send + Sync> Message for T {}
@ -32,6 +57,11 @@ pub trait ActorInterface: 'static + Send {
/// If your `Incoming` type IS `Down`, this method is never called — the /// If your `Incoming` type IS `Down`, this method is never called — the
/// normal `handle()` receives the message instead. /// normal `handle()` receives the message instead.
fn handle_down(&mut self, _ctx: &Ctx, _down: Down) {} fn handle_down(&mut self, _ctx: &Ctx, _down: Down) {}
/// Called when a watched actor exits. Override to react to death notifications.
///
/// Default: no-op (notification is silently consumed).
fn on_actor_exit(&mut self, _ctx: &Ctx, _exited: ActorExited) {}
} }
/// A unique address for this actor. 32 bytes is overkill for a small application, /// A unique address for this actor. 32 bytes is overkill for a small application,
@ -106,10 +136,17 @@ where
} }
Err(msg) => msg, Err(msg) => msg,
}; };
match msg.downcast::<Down>() { let msg = match msg.downcast::<Down>() {
Ok(down) => { Ok(down) => {
self.inner.handle_down(ctx, *down); self.inner.handle_down(ctx, *down);
Some("swactor::actor::Down") return Some("swactor::actor::Down");
}
Err(msg) => msg,
};
match msg.downcast::<ActorExited>() {
Ok(exited) => {
self.inner.on_actor_exit(ctx, *exited);
Some("ActorExited")
} }
Err(_) => None, Err(_) => None,
} }
@ -205,6 +242,10 @@ pub trait ContextInner {
fn schedule_timer(&self, request: TimerRequest); fn schedule_timer(&self, request: TimerRequest);
/// Access the runtime extension (if installed). /// Access the runtime extension (if installed).
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>; fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>;
/// Register a watch: watcher receives ActorExited when target dies.
fn watch(&self, watcher: ActorAddress, target: ActorAddress);
/// Cancel a watch.
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress);
} }
/// Actor syscall interface — passed to `ActorInterface::handle()`. /// Actor syscall interface — passed to `ActorInterface::handle()`.
@ -290,4 +331,22 @@ impl<'a> Ctx<'a> {
period, period,
}); });
} }
/// Watch another actor's liveness. If the target dies, this actor
/// receives an `ActorExited` message in its mailbox.
///
/// Watching an already-dead or non-existent actor delivers
/// `ActorExited { reason: Stopped }` on the next tick.
///
/// Calling watch() multiple times on the same target is idempotent —
/// only one notification is delivered.
pub fn watch(&self, target: ActorAddress) {
self.inner.watch(self.self_addr, target);
}
/// Stop watching an actor. No notification will be delivered if the
/// target subsequently dies.
pub fn unwatch(&self, target: ActorAddress) {
self.inner.unwatch(self.self_addr, target);
}
} }

View file

@ -2,13 +2,14 @@ use std::any::Any;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::hash::{BuildHasher, Hasher}; use std::hash::{BuildHasher, Hasher};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock, RwLock}; use std::sync::{Arc, Mutex, OnceLock, RwLock};
use std::thread::Thread; use std::thread::Thread;
use crate::actor::{ActorAddress, AnyActor, Message}; use crate::actor::{ActorAddress, AnyActor, Message};
use crate::channel::Sender; use crate::channel::Sender;
use crate::config::RuntimeConfig; use crate::config::RuntimeConfig;
use crate::stats::WorkerStats; use crate::stats::WorkerStats;
use crate::worker::WatchRegistry;
use crate::Error; use crate::Error;
// ─── Identity Hasher for ActorAddress ─────────────────────────────────────── // ─── Identity Hasher for ActorAddress ───────────────────────────────────────
@ -243,6 +244,7 @@ pub(crate) struct TickContext<'a> {
pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>, pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>,
/// Thread handles for waking parked workers on cross-worker sends. /// Thread handles for waking parked workers on cross-worker sends.
pub(crate) worker_threads: &'a [OnceLock<Thread>], pub(crate) worker_threads: &'a [OnceLock<Thread>],
pub(crate) watch_registry: Option<&'a Arc<Mutex<WatchRegistry>>>,
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>, pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>,
#[cfg(feature = "transport")] #[cfg(feature = "transport")]

View file

@ -1,11 +1,11 @@
use std::any::Any; use std::any::Any;
use std::cell::RefCell; use std::cell::RefCell;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, Mutex, OnceLock};
use std::thread::{self, JoinHandle, Thread}; use std::thread::{self, JoinHandle, Thread};
use std::time::Instant; use std::time::Instant;
use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message, StopSignal, TimerRequest}; use crate::actor::{Actor, ActorAddress, ActorExited, ActorInterface, AnyActor, ExitReason, Message, StopSignal, TimerRequest};
use crate::channel::{Receiver, Sender}; use crate::channel::{Receiver, Sender};
// Re-export config types so existing code using `runtime::RuntimeConfig` still works // Re-export config types so existing code using `runtime::RuntimeConfig` still works
pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig}; pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig};
@ -14,7 +14,7 @@ use crate::extension::RuntimeExtension;
use crate::stats::{StatsHook, WorkerStats}; use crate::stats::{StatsHook, WorkerStats};
// Re-export stats types so existing code using `runtime::*` still works // Re-export stats types so existing code using `runtime::*` still works
pub use crate::stats::{RuntimeStats, WorkerInfo}; pub use crate::stats::{RuntimeStats, WorkerInfo};
use crate::worker::Worker; use crate::worker::{WatchRegistry, Worker};
use crate::Error; use crate::Error;
/// Generic message inbox for receiving messages outside of the runtime. /// Generic message inbox for receiving messages outside of the runtime.
@ -103,6 +103,7 @@ pub struct Runtime {
is_running: AtomicBool, is_running: AtomicBool,
worker_stats: Vec<Arc<WorkerStats>>, worker_stats: Vec<Arc<WorkerStats>>,
stats_hook: Option<Arc<dyn StatsHook>>, stats_hook: Option<Arc<dyn StatsHook>>,
watch_registry: Arc<Mutex<WatchRegistry>>,
/// Workers available for tick(). run() drains this and moves workers to threads. /// Workers available for tick(). run() drains this and moves workers to threads.
tick_workers: RefCell<Vec<Worker>>, tick_workers: RefCell<Vec<Worker>>,
/// Thread handles for waking parked workers. Set by workers on startup via OnceLock. /// Thread handles for waking parked workers. Set by workers on startup via OnceLock.
@ -198,6 +199,7 @@ impl Runtime {
is_running: AtomicBool::new(false), is_running: AtomicBool::new(false),
worker_stats, worker_stats,
stats_hook: None, stats_hook: None,
watch_registry: Arc::new(Mutex::new(WatchRegistry::new())),
tick_workers: RefCell::new(workers), tick_workers: RefCell::new(workers),
worker_threads, worker_threads,
created_at: Instant::now(), created_at: Instant::now(),
@ -304,6 +306,7 @@ impl Runtime {
extension: self.extension.as_deref(), extension: self.extension.as_deref(),
stats_hook: self.stats_hook.as_deref(), stats_hook: self.stats_hook.as_deref(),
worker_threads: &self.worker_threads, worker_threads: &self.worker_threads,
watch_registry: Some(&self.watch_registry),
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
codec_registry: self.codec_registry.as_deref(), codec_registry: self.codec_registry.as_deref(),
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
@ -506,4 +509,25 @@ impl ContextInner for Runtime {
fn extension(&self) -> Option<&dyn RuntimeExtension> { fn extension(&self) -> Option<&dyn RuntimeExtension> {
self.extension.as_deref() self.extension.as_deref()
} }
fn watch(&self, watcher: ActorAddress, target: ActorAddress) {
if self.address_map.lookup(&target).is_some() {
self.watch_registry.lock().unwrap().watch(watcher, target);
} else {
// Target not found — deliver ActorExited immediately.
let msg = ActorExited {
addr: target,
reason: ExitReason::Stopped,
};
// Route to watcher via transfer queue
if let Some(wid) = self.address_map.lookup(&watcher) {
self.transfer_txs[wid.as_usize()]
.send(Envelope::new(watcher, Box::new(msg)));
}
}
}
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) {
self.watch_registry.lock().unwrap().unwatch(watcher, target);
}
} }

View file

@ -1,12 +1,12 @@
use std::any::Any; use std::any::Any;
use std::cell::RefCell; use std::cell::RefCell;
use std::collections::{HashMap, VecDeque}; use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::{Arc, Mutex};
use std::thread; use std::thread;
use std::time::Instant; use std::time::Instant;
use crate::actor::{ActorAddress, AnyActor, CloneMsg, ContextInner, Ctx, StopReason, StopSignal, TimerRequest}; use crate::actor::{ActorAddress, ActorExited, AnyActor, CloneMsg, ContextInner, Ctx, ExitReason, StopReason, StopSignal, TimerRequest};
use crate::channel::Receiver; use crate::channel::Receiver;
use crate::config::MailboxOverflow; use crate::config::MailboxOverflow;
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId}; use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
@ -108,6 +108,95 @@ impl TimerWheel {
} }
} }
// ─── Watch Registry ─────────────────────────────────────────────────────────
/// Tracks watch relationships between actors.
///
/// Shared across workers via `Arc<Mutex<_>>`. Contention is negligible
/// because watch/unwatch operations are rare relative to message sends.
pub(crate) struct WatchRegistry {
/// target → set of watchers awaiting death notification
watchers: HashMap<ActorAddress, HashSet<ActorAddress>>,
/// watcher → set of targets it's watching (reverse index for cleanup)
watching: HashMap<ActorAddress, HashSet<ActorAddress>>,
}
impl WatchRegistry {
pub fn new() -> Self {
Self {
watchers: HashMap::new(),
watching: HashMap::new(),
}
}
pub fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) {
self.watchers.entry(target).or_default().insert(watcher);
self.watching.entry(watcher).or_default().insert(target);
}
pub fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) {
if let Some(set) = self.watchers.get_mut(&target) {
set.remove(&watcher);
if set.is_empty() {
self.watchers.remove(&target);
}
}
if let Some(set) = self.watching.get_mut(&watcher) {
set.remove(&target);
if set.is_empty() {
self.watching.remove(&watcher);
}
}
}
/// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs.
pub fn notify_death(
&mut self,
target: ActorAddress,
reason: ExitReason,
) -> Vec<(ActorAddress, ActorExited)> {
let notification = ActorExited {
addr: target,
reason,
};
let mut result = Vec::new();
if let Some(watcher_set) = self.watchers.remove(&target) {
for watcher in &watcher_set {
result.push((*watcher, notification.clone()));
// clean up reverse index
if let Some(set) = self.watching.get_mut(watcher) {
set.remove(&target);
if set.is_empty() {
self.watching.remove(watcher);
}
}
}
}
result
}
/// Called when a watcher itself dies. Cleans up all its watching entries.
pub fn cleanup_watcher(&mut self, watcher: &ActorAddress) {
if let Some(targets) = self.watching.remove(watcher) {
for target in targets {
if let Some(set) = self.watchers.get_mut(&target) {
set.remove(watcher);
if set.is_empty() {
self.watchers.remove(&target);
}
}
}
}
}
/// Check if a target has any watchers registered.
pub fn has_watchers(&self, target: &ActorAddress) -> bool {
self.watchers.get(target).is_some_and(|s| !s.is_empty())
}
}
// ─── Worker ───────────────────────────────────────────────────────────────── // ─── Worker ─────────────────────────────────────────────────────────────────
/// A worker owns a set of actors and runs them in a loop. /// A worker owns a set of actors and runs them in a loop.
@ -203,6 +292,7 @@ impl Worker {
let timer_requests: RefCell<Vec<TimerRequest>> = RefCell::new(Vec::new()); let timer_requests: RefCell<Vec<TimerRequest>> = RefCell::new(Vec::new());
let processed; let processed;
let deaths;
{ {
let worker_ctx = WorkerContext { let worker_ctx = WorkerContext {
worker_id: self.id, worker_id: self.id,
@ -212,7 +302,7 @@ impl Worker {
timer_requests: &timer_requests, timer_requests: &timer_requests,
stats: &self.stats, stats: &self.stats,
}; };
processed = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests); (processed, deaths) = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests);
if processed > 0 { if processed > 0 {
did_work = true; did_work = true;
} }
@ -256,6 +346,40 @@ impl Worker {
} }
} }
} }
// 5b. Process actor deaths → deliver ActorExited to watchers
if !deaths.is_empty() {
did_work = true;
if let Some(registry) = &tc.watch_registry {
let mut reg = registry.lock().unwrap();
for (dead_addr, reason) in deaths {
let notifications = reg.notify_death(dead_addr, reason);
for (watcher_addr, msg) in notifications {
// Deliver ActorExited as a normal message via the address map
match tc.address_map.lookup(&watcher_addr) {
Some(wid) if wid == self.id => {
self.pool.deliver(&watcher_addr, Box::new(msg));
}
Some(wid) => {
tc.transfer_txs[wid.as_usize()]
.send(Envelope::new(watcher_addr, Box::new(msg)));
}
None => {
// Watcher not in address map — may be an inbox or remote.
// Try inbox registry as best effort.
let _ = tc.inbox_registry.try_deliver(
watcher_addr,
Box::new(msg),
);
}
}
}
// Clean up the dead actor's own watches (things it was watching)
reg.cleanup_watcher(&dead_addr);
}
}
}
let t5 = Instant::now(); let t5 = Instant::now();
// 6. Publish stats (skip entirely when idle to avoid allocation + mutex) // 6. Publish stats (skip entirely when idle to avoid allocation + mutex)
@ -451,6 +575,29 @@ impl ContextInner for WorkerContext<'_> {
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> { fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
self.tc.extension self.tc.extension
} }
fn watch(&self, watcher: ActorAddress, target: ActorAddress) {
if let Some(registry) = &self.tc.watch_registry {
// Check if target exists in the address map
if self.tc.address_map.lookup(&target).is_some() {
registry.lock().unwrap().watch(watcher, target);
} else {
// Target not found — deliver ActorExited { reason: Stopped } immediately.
// Buffer in pending_local so it arrives on next tick.
let msg = ActorExited {
addr: target,
reason: ExitReason::Stopped,
};
self.pending_local.borrow_mut().push((watcher, Box::new(msg)));
}
}
}
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) {
if let Some(registry) = &self.tc.watch_registry {
registry.lock().unwrap().unwatch(watcher, target);
}
}
} }
struct ActorSlot { struct ActorSlot {
@ -531,7 +678,7 @@ impl ActorPool {
std::mem::replace(&mut self.drops_this_tick, 0) std::mem::replace(&mut self.drops_this_tick, 0)
} }
/// Tick all actors in the pool. Returns the number of messages processed. /// Tick all actors in the pool. Returns (messages_processed, newly_dead_actors).
/// ///
/// Each actor processes up to `budget` messages per tick (0 = unlimited). /// Each actor processes up to `budget` messages per tick (0 = unlimited).
/// This prevents a single hot actor from starving others on the same worker. /// This prevents a single hot actor from starving others on the same worker.
@ -541,8 +688,9 @@ impl ActorPool {
stats: &WorkerStats, stats: &WorkerStats,
budget: usize, budget: usize,
stop_requests: &RefCell<Vec<ActorAddress>>, stop_requests: &RefCell<Vec<ActorAddress>>,
) -> usize { ) -> (usize, Vec<(ActorAddress, ExitReason)>) {
let mut count = 0; let mut count = 0;
let mut deaths = Vec::new();
for (&addr, slot) in self.actors.iter_mut() { for (&addr, slot) in self.actors.iter_mut() {
if slot.poisoned || slot.stopping { if slot.poisoned || slot.stopping {
// Discard all messages for poisoned/stopping actors // Discard all messages for poisoned/stopping actors
@ -606,6 +754,8 @@ impl ActorPool {
#[cfg(feature = "tracing")] #[cfg(feature = "tracing")]
tracing::error!(actor_addr = %addr, "actor.panicked"); tracing::error!(actor_addr = %addr, "actor.panicked");
slot.poisoned = true; slot.poisoned = true;
slot.mailbox.clear();
deaths.push((addr, ExitReason::Panicked));
break; break;
} }
Ok(Some(type_name)) => { Ok(Some(type_name)) => {
@ -624,6 +774,7 @@ impl ActorPool {
slot.stopping = true; slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed); stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear(); slot.mailbox.clear();
deaths.push((addr, ExitReason::Stopped));
break; break;
} }
} }
@ -633,7 +784,7 @@ impl ActorPool {
} }
} }
} }
count (count, deaths)
} }
pub fn len(&self) -> usize { pub fn len(&self) -> usize {

378
tests/watch_api.rs Normal file
View file

@ -0,0 +1,378 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use swactor::actor::{ActorAddress, ActorExited, ActorInterface, ExitReason};
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
// ── Actors ──────────────────────────────────────────────────────────────────
/// An actor that panics when it receives PanicMsg.
struct PanicOnCommand;
#[derive(Clone)]
struct PanicMsg;
impl ActorInterface for PanicOnCommand {
type Incoming = PanicMsg;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: PanicMsg) {
panic!("deliberate panic for test");
}
}
/// An actor that watches targets and counts exit notifications.
struct ExitWatcher {
exit_count: Arc<AtomicUsize>,
last_reason: Arc<std::sync::Mutex<Option<ExitReason>>>,
last_addr: Arc<std::sync::Mutex<Option<ActorAddress>>>,
}
#[derive(Clone)]
enum WatcherCmd {
WatchThis(ActorAddress),
UnwatchThis(ActorAddress),
}
impl ActorInterface for ExitWatcher {
type Incoming = WatcherCmd;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: WatcherCmd) {
match msg {
WatcherCmd::WatchThis(target) => {
ctx.watch(target);
}
WatcherCmd::UnwatchThis(target) => {
ctx.unwatch(target);
}
}
}
fn on_actor_exit(&mut self, _ctx: &Ctx, exited: ActorExited) {
self.exit_count.fetch_add(1, Ordering::SeqCst);
*self.last_reason.lock().unwrap() = Some(exited.reason);
*self.last_addr.lock().unwrap() = Some(exited.addr);
}
}
impl ExitWatcher {
fn new() -> (Self, WatcherState) {
let exit_count = Arc::new(AtomicUsize::new(0));
let last_reason = Arc::new(std::sync::Mutex::new(None));
let last_addr = Arc::new(std::sync::Mutex::new(None));
let state = WatcherState {
exit_count: exit_count.clone(),
last_reason: last_reason.clone(),
last_addr: last_addr.clone(),
};
(
ExitWatcher {
exit_count,
last_reason,
last_addr,
},
state,
)
}
}
/// Shared state for inspecting what ExitWatcher observed.
struct WatcherState {
exit_count: Arc<AtomicUsize>,
last_reason: Arc<std::sync::Mutex<Option<ExitReason>>>,
last_addr: Arc<std::sync::Mutex<Option<ActorAddress>>>,
}
impl WatcherState {
fn count(&self) -> usize {
self.exit_count.load(Ordering::SeqCst)
}
fn last_reason(&self) -> Option<ExitReason> {
self.last_reason.lock().unwrap().clone()
}
fn last_addr(&self) -> Option<ActorAddress> {
*self.last_addr.lock().unwrap()
}
}
/// A silent actor that does nothing (for targets that shouldn't panic).
struct Sleeper;
#[derive(Clone)]
struct Noop;
impl ActorInterface for Sleeper {
type Incoming = Noop;
type Response = ();
fn handle(&mut self, _ctx: &Ctx, _msg: Noop) {}
}
// ── Helper ──────────────────────────────────────────────────────────────────
fn tick_n(rt: &Runtime, n: usize) {
for _ in 0..n {
rt.tick();
}
}
fn single_thread_config() -> RuntimeConfig {
RuntimeConfig {
num_threads: 1,
..RuntimeConfig::default()
}
}
// ── Tests ───────────────────────────────────────────────────────────────────
/// Given a watcher and a target actor,
/// when the target panics,
/// then the watcher's on_actor_exit fires with ExitReason::Panicked.
#[test]
fn watch_receives_notification_on_panic() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
// Tell watcher to watch the target
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill the target
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "watcher should have received exactly one ActorExited");
assert_eq!(state.last_reason(), Some(ExitReason::Panicked));
assert_eq!(state.last_addr(), Some(target));
}
/// Given a watcher that watches then unwatches a target,
/// when the target panics,
/// then the watcher receives NO notification.
#[test]
fn unwatch_prevents_notification() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
// Watch
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Unwatch
rt.send_to(watcher, WatcherCmd::UnwatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill target
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 0, "after unwatch, no notification should be delivered");
}
/// Given a watch on an address that was never spawned,
/// then the watcher receives ActorExited { reason: Stopped }.
#[test]
fn watch_nonexistent_actor_delivers_stopped() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let watcher = rt.spawn(watcher_actor).unwrap();
let nonexistent = ActorAddress::new_random();
rt.send_to(watcher, WatcherCmd::WatchThis(nonexistent)).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "should receive ActorExited for non-existent target");
assert_eq!(state.last_reason(), Some(ExitReason::Stopped));
assert_eq!(state.last_addr(), Some(nonexistent));
}
/// Given a watcher that dies before the target,
/// when the target subsequently panics,
/// then there is no panic or leak.
#[test]
fn watcher_dies_before_target_no_panic() {
let rt = Runtime::new(single_thread_config());
let target = rt.spawn(PanicOnCommand).unwrap();
let (watcher_actor, _state) = ExitWatcher::new();
let watcher = rt.spawn(watcher_actor).unwrap();
// Watch
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill the watcher first (send it a type-mismatched panic msg directly)
// Actually, ExitWatcher doesn't panic. Use Runtime-level watch + PanicOnCommand.
let rt2 = Runtime::new(single_thread_config());
let target2 = rt2.spawn(PanicOnCommand).unwrap();
let watcher2 = rt2.spawn(PanicOnCommand).unwrap();
use swactor::actor::ContextInner;
rt2.watch(watcher2, target2);
tick_n(&rt2, 3);
// Kill watcher first
rt2.send_to(watcher2, PanicMsg).unwrap();
tick_n(&rt2, 5);
// Kill target — should not crash
rt2.send_to(target2, PanicMsg).unwrap();
tick_n(&rt2, 5);
// If we got here, no crash.
}
/// Given a watcher that calls watch() twice on the same target,
/// when the target panics,
/// then the watcher receives exactly one notification.
#[test]
fn idempotent_watch_delivers_one_notification() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
// Watch twice
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
// Kill target
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "double watch should produce exactly one notification");
}
/// Given multiple watchers on the same target,
/// when the target panics,
/// then all watchers receive the notification.
#[test]
fn multiple_watchers_all_notified() {
let rt = Runtime::new(single_thread_config());
let (w1_actor, s1) = ExitWatcher::new();
let (w2_actor, s2) = ExitWatcher::new();
let (w3_actor, s3) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let w1 = rt.spawn(w1_actor).unwrap();
let w2 = rt.spawn(w2_actor).unwrap();
let w3 = rt.spawn(w3_actor).unwrap();
rt.send_to(w1, WatcherCmd::WatchThis(target)).unwrap();
rt.send_to(w2, WatcherCmd::WatchThis(target)).unwrap();
rt.send_to(w3, WatcherCmd::WatchThis(target)).unwrap();
tick_n(&rt, 3);
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(s1.count(), 1, "watcher 1 should be notified");
assert_eq!(s2.count(), 1, "watcher 2 should be notified");
assert_eq!(s3.count(), 1, "watcher 3 should be notified");
}
/// Self-watch doesn't crash the runtime.
#[test]
fn self_watch_does_not_crash() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, _state) = ExitWatcher::new();
let actor = rt.spawn(watcher_actor).unwrap();
rt.send_to(actor, WatcherCmd::WatchThis(actor)).unwrap();
tick_n(&rt, 5);
// No crash = pass
}
/// Runtime-level watch (outside actor context) delivers notification.
#[test]
fn runtime_level_watch_delivers_notification() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let target = rt.spawn(PanicOnCommand).unwrap();
let watcher = rt.spawn(watcher_actor).unwrap();
tick_n(&rt, 2); // ensure both spawned
use swactor::actor::ContextInner;
rt.watch(watcher, target);
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "runtime-level watch should deliver notification");
assert_eq!(state.last_reason(), Some(ExitReason::Panicked));
}
/// Runtime-level watch on non-existent address delivers Stopped.
#[test]
fn runtime_level_watch_nonexistent_delivers_stopped() {
let rt = Runtime::new(single_thread_config());
let (watcher_actor, state) = ExitWatcher::new();
let watcher = rt.spawn(watcher_actor).unwrap();
tick_n(&rt, 2);
let fake = ActorAddress::new_random();
use swactor::actor::ContextInner;
rt.watch(watcher, fake);
tick_n(&rt, 5);
assert_eq!(state.count(), 1, "watching non-existent from runtime should deliver Stopped");
assert_eq!(state.last_reason(), Some(ExitReason::Stopped));
}
/// Given a watcher watching target via on_actor_exit,
/// when target panics,
/// then the watcher can react by spawning a replacement (supervision pattern).
#[test]
fn watcher_can_react_to_death_by_spawning() {
let rt = Runtime::new(single_thread_config());
let spawned = Arc::new(AtomicUsize::new(0));
struct Supervisor {
spawned_count: Arc<AtomicUsize>,
}
#[derive(Clone)]
enum SupervisorMsg {
WatchThis(ActorAddress),
}
impl ActorInterface for Supervisor {
type Incoming = SupervisorMsg;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) {
match msg {
SupervisorMsg::WatchThis(target) => ctx.watch(target),
}
}
fn on_actor_exit(&mut self, ctx: &Ctx, _exited: ActorExited) {
// React: spawn a replacement
let replacement = ctx.spawn(Sleeper).unwrap();
let _ = replacement;
self.spawned_count.fetch_add(1, Ordering::SeqCst);
}
}
let target = rt.spawn(PanicOnCommand).unwrap();
let sup = rt.spawn(Supervisor { spawned_count: spawned.clone() }).unwrap();
rt.send_to(sup, SupervisorMsg::WatchThis(target)).unwrap();
tick_n(&rt, 3);
rt.send_to(target, PanicMsg).unwrap();
tick_n(&rt, 5);
assert_eq!(spawned.load(Ordering::SeqCst), 1, "supervisor should have spawned a replacement");
}