feat: wasm runner actor skeleton (#32)
Lay down the wasm-actor host, a frontend-agnostic command layer, and a distribution registry. - command (new crate): CommandRouter dispatching to built-in inspection handlers (overview/workers/actors) plus user-registered handlers, with line and query-param parsers; built for REPL/REST/TUI/WebSocket frontends. - wasm-actor (new crate): skeleton host — WasmActor, Builder, Engine, error types — with echo/double/silent guest fixtures and integration tests. - distribution: add Registry (member catalog + lookups) and Snapshot, with tests. - core: extend the worker watch API; add watch_api integration tests. Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
parent
3c29293945
commit
4e56590f05
43 changed files with 5049 additions and 644 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -11,3 +11,7 @@ corpus
|
|||
**/deps.html
|
||||
docs/architecture.dot
|
||||
docs/architecture.html
|
||||
|
||||
# Claude session files
|
||||
CLAUDE/
|
||||
.claude/
|
||||
|
|
|
|||
1083
Cargo.lock
generated
1083
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,5 +1,5 @@
|
|||
[workspace]
|
||||
members = [".", "crates/python", "crates/wasm", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std"]
|
||||
members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std", "crates/command"]
|
||||
exclude = ["tools/depgraph"]
|
||||
|
||||
[package]
|
||||
|
|
|
|||
9
crates/command/Cargo.toml
Normal file
9
crates/command/Cargo.toml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "swactor-command"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
swactor = { path = "../..", features = ["serde"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
561
crates/command/src/builtins.rs
Normal file
561
crates/command/src/builtins.rs
Normal file
|
|
@ -0,0 +1,561 @@
|
|||
//! Built-in command handlers for runtime inspection and management.
|
||||
//!
|
||||
//! Extracted from `crates/runtime-dashboard/src/investigate.rs`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::stats::TickTiming;
|
||||
|
||||
use crate::{CommandContext, CommandHandler, CommandMeta, CommandResponse};
|
||||
|
||||
// ─── Arg helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
fn arg_str<'a>(args: &'a HashMap<String, serde_json::Value>, key: &str) -> Option<&'a str> {
|
||||
args.get(key).and_then(|v| v.as_str())
|
||||
}
|
||||
|
||||
fn arg_usize(args: &HashMap<String, serde_json::Value>, key: &str) -> Option<usize> {
|
||||
args.get(key).and_then(|v| {
|
||||
v.as_u64()
|
||||
.map(|n| n as usize)
|
||||
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
|
||||
})
|
||||
}
|
||||
|
||||
fn arg_f64(args: &HashMap<String, serde_json::Value>, key: &str) -> Option<f64> {
|
||||
args.get(key).and_then(|v| {
|
||||
v.as_f64()
|
||||
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Display helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
fn format_addr(addr: &ActorAddress) -> String {
|
||||
format!("{addr}")
|
||||
}
|
||||
|
||||
fn full_hex(addr: &ActorAddress) -> String {
|
||||
addr.0.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
// ─── Phase breakdown helper ──────────────────────────────────────────────────
|
||||
|
||||
fn compute_phase_breakdown(timings: &[TickTiming]) -> serde_json::Value {
|
||||
if timings.is_empty() {
|
||||
return serde_json::json!({
|
||||
"ticks": 0,
|
||||
"active_pct": 0.0,
|
||||
"avg_tick_us": 0.0,
|
||||
"phases_us": [0, 0, 0, 0, 0, 0],
|
||||
"phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
||||
});
|
||||
}
|
||||
|
||||
let n = timings.len();
|
||||
let active = timings.iter().filter(|t| t.did_work).count();
|
||||
let active_pct = (active as f64 / n as f64) * 100.0;
|
||||
|
||||
let mut phase_sums = [0u64; 6];
|
||||
for t in timings {
|
||||
for (i, &us) in t.phase_us.iter().enumerate() {
|
||||
phase_sums[i] += us;
|
||||
}
|
||||
}
|
||||
let total_us: u64 = phase_sums.iter().sum();
|
||||
let avg_tick_us = total_us as f64 / n as f64;
|
||||
|
||||
let phases_pct: Vec<f64> = if total_us == 0 {
|
||||
vec![0.0; 6]
|
||||
} else {
|
||||
phase_sums
|
||||
.iter()
|
||||
.map(|&s| (s as f64 / total_us as f64) * 100.0)
|
||||
.collect()
|
||||
};
|
||||
|
||||
serde_json::json!({
|
||||
"ticks": n,
|
||||
"active_pct": active_pct,
|
||||
"avg_tick_us": avg_tick_us,
|
||||
"phases_us": phase_sums,
|
||||
"phases_pct": phases_pct,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Read Commands ───────────────────────────────────────────────────────────
|
||||
|
||||
pub struct OverviewCommand;
|
||||
|
||||
impl CommandHandler for OverviewCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "overview",
|
||||
description: "Summary: worker count, actor count, total messages, mailbox depth, panics",
|
||||
usage: "overview",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
_args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let stats = ctx.enriched_stats();
|
||||
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
|
||||
let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum();
|
||||
let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum();
|
||||
let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum();
|
||||
let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum();
|
||||
let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum();
|
||||
|
||||
CommandResponse::ok(
|
||||
"overview",
|
||||
serde_json::json!({
|
||||
"workers": stats.num_workers,
|
||||
"actors": stats.actor_details.len(),
|
||||
"total_messages_processed": total_msgs,
|
||||
"total_mailbox_depth": total_mailbox,
|
||||
"total_panics": total_panics,
|
||||
"total_type_mismatches": total_type_mismatches,
|
||||
"sends": {
|
||||
"local": total_local,
|
||||
"cross_worker": total_cross,
|
||||
"inbox": total_inbox,
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WorkersCommand;
|
||||
|
||||
impl CommandHandler for WorkersCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "workers",
|
||||
description: "Per-worker stats: actors, mailbox depth, messages, sends, panics",
|
||||
usage: "workers",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
_args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let stats = ctx.stats();
|
||||
let workers: Vec<_> = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|w| {
|
||||
serde_json::json!({
|
||||
"id": w.id,
|
||||
"actors": w.num_actors,
|
||||
"mailbox_depth": w.mailbox_depth,
|
||||
"messages_processed": w.messages_processed,
|
||||
"local_sends": w.local_sends,
|
||||
"cross_sends": w.cross_sends,
|
||||
"inbox_sends": w.inbox_sends,
|
||||
"type_mismatches": w.type_mismatches,
|
||||
"panics": w.panics,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
CommandResponse::ok("workers", workers)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WorkerCommand;
|
||||
|
||||
impl CommandHandler for WorkerCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "worker",
|
||||
description: "Single worker detail with tick-phase timing breakdown",
|
||||
usage: "worker <id>",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let id = match arg_usize(args, "id") {
|
||||
Some(id) => id,
|
||||
None => return CommandResponse::err("worker", "usage: worker <id>"),
|
||||
};
|
||||
|
||||
let stats = ctx.enriched_stats();
|
||||
let w = match stats.workers.iter().find(|w| w.id == id) {
|
||||
Some(w) => w,
|
||||
None => {
|
||||
return CommandResponse::err(
|
||||
"worker",
|
||||
format!("worker {id} not found (have 0..{})", stats.num_workers),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let timings = stats.tick_timings.get(id).cloned().unwrap_or_default();
|
||||
let phase_breakdown = compute_phase_breakdown(&timings);
|
||||
|
||||
let actors_on_worker: Vec<_> = stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.filter(|a| a.worker_id == id)
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CommandResponse::ok(
|
||||
"worker",
|
||||
serde_json::json!({
|
||||
"id": w.id,
|
||||
"actors": w.num_actors,
|
||||
"mailbox_depth": w.mailbox_depth,
|
||||
"messages_processed": w.messages_processed,
|
||||
"local_sends": w.local_sends,
|
||||
"cross_sends": w.cross_sends,
|
||||
"inbox_sends": w.inbox_sends,
|
||||
"type_mismatches": w.type_mismatches,
|
||||
"panics": w.panics,
|
||||
"tick_phases": phase_breakdown,
|
||||
"actor_details": actors_on_worker,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ActorsCommand;
|
||||
|
||||
impl CommandHandler for ActorsCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "actors",
|
||||
description: "List actors with optional sorting, limit, and worker filter",
|
||||
usage: "actors [--sort mailbox|worker|address] [--limit N] [--worker W]",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let stats = ctx.enriched_stats();
|
||||
let mut actors = stats.actor_details.clone();
|
||||
|
||||
let sort_by = arg_str(args, "sort").unwrap_or("mailbox");
|
||||
let limit = arg_usize(args, "limit").unwrap_or(usize::MAX);
|
||||
let worker_filter = arg_usize(args, "worker");
|
||||
|
||||
if let Some(wid) = worker_filter {
|
||||
actors.retain(|a| a.worker_id == wid);
|
||||
}
|
||||
|
||||
match sort_by {
|
||||
"mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)),
|
||||
"worker" => actors.sort_by_key(|a| a.worker_id),
|
||||
"address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)),
|
||||
other => {
|
||||
return CommandResponse::err(
|
||||
"actors",
|
||||
format!("unknown sort field `{other}` — use mailbox|worker|address"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
actors.truncate(limit);
|
||||
|
||||
let rows: Vec<_> = actors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"address_full": full_hex(&a.address),
|
||||
"worker_id": a.worker_id,
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CommandResponse::ok(
|
||||
"actors",
|
||||
serde_json::json!({
|
||||
"total": stats.actor_details.len(),
|
||||
"returned": rows.len(),
|
||||
"sort": sort_by,
|
||||
"actors": rows,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ActorCommand;
|
||||
|
||||
impl CommandHandler for ActorCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "actor",
|
||||
description: "Find actor(s) whose address starts with the given hex prefix",
|
||||
usage: "actor <hex_prefix>",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let prefix = match arg_str(args, "prefix") {
|
||||
Some(p) => p,
|
||||
None => return CommandResponse::err("actor", "usage: actor <hex_prefix>"),
|
||||
};
|
||||
|
||||
let stats = ctx.enriched_stats();
|
||||
let matches: Vec<_> = stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.filter(|a| full_hex(&a.address).starts_with(prefix))
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"address_full": full_hex(&a.address),
|
||||
"worker_id": a.worker_id,
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CommandResponse::ok(
|
||||
"actor",
|
||||
serde_json::json!({
|
||||
"prefix": prefix,
|
||||
"matches": matches.len(),
|
||||
"actors": matches,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HotCommand;
|
||||
|
||||
impl CommandHandler for HotCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "hot",
|
||||
description: "Top N actors by mailbox depth (default 10)",
|
||||
usage: "hot [N]",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let n = arg_usize(args, "n").unwrap_or(10);
|
||||
let stats = ctx.enriched_stats();
|
||||
|
||||
let mut actors = stats.actor_details.clone();
|
||||
actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth));
|
||||
actors.truncate(n);
|
||||
|
||||
let rows: Vec<_> = actors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"address_full": full_hex(&a.address),
|
||||
"worker_id": a.worker_id,
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CommandResponse::ok("hot", rows)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PhasesCommand;
|
||||
|
||||
impl CommandHandler for PhasesCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "phases",
|
||||
description: "Tick-phase time breakdown (all workers or one)",
|
||||
usage: "phases [worker_id]",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let stats = ctx.stats();
|
||||
let worker_filter = arg_usize(args, "worker");
|
||||
|
||||
let phase_names = [
|
||||
"spawn_drain",
|
||||
"transfer_drain",
|
||||
"tick_all",
|
||||
"spawn_drain_2",
|
||||
"pending_local",
|
||||
"stats_publish",
|
||||
];
|
||||
|
||||
let mut results = Vec::new();
|
||||
for (i, timings) in stats.tick_timings.iter().enumerate() {
|
||||
if let Some(wid) = worker_filter {
|
||||
if i != wid {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let breakdown = compute_phase_breakdown(timings);
|
||||
results.push(serde_json::json!({
|
||||
"worker_id": i,
|
||||
"ticks_sampled": timings.len(),
|
||||
"phases": breakdown,
|
||||
"phase_names": phase_names,
|
||||
}));
|
||||
}
|
||||
|
||||
CommandResponse::ok("phases", results)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DiffCommand;
|
||||
|
||||
impl CommandHandler for DiffCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "diff",
|
||||
description: "Collect two snapshots N seconds apart, report deltas and rates",
|
||||
usage: "diff <seconds>",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
let secs = match arg_f64(args, "seconds") {
|
||||
Some(s) if s > 0.0 && s <= 30.0 => s,
|
||||
Some(_) => return CommandResponse::err("diff", "seconds must be between 0 and 30"),
|
||||
None => return CommandResponse::err("diff", "usage: diff <seconds>"),
|
||||
};
|
||||
|
||||
let before = ctx.enriched_stats();
|
||||
let t0 = Instant::now();
|
||||
std::thread::sleep(Duration::from_secs_f64(secs));
|
||||
let after = ctx.enriched_stats();
|
||||
let elapsed = t0.elapsed().as_secs_f64();
|
||||
|
||||
let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum();
|
||||
let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum();
|
||||
let delta_msgs = msgs_after.saturating_sub(msgs_before);
|
||||
|
||||
let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum();
|
||||
let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum();
|
||||
let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum();
|
||||
let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum();
|
||||
|
||||
let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
|
||||
let per_worker: Vec<_> = after
|
||||
.workers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, w)| {
|
||||
let prev = before.workers.get(i);
|
||||
let d = prev
|
||||
.map(|p| w.messages_processed.saturating_sub(p.messages_processed))
|
||||
.unwrap_or(0);
|
||||
serde_json::json!({
|
||||
"worker_id": i,
|
||||
"delta_messages": d,
|
||||
"msg_per_sec": d as f64 / elapsed,
|
||||
"actors_before": prev.map(|p| p.num_actors).unwrap_or(0),
|
||||
"actors_after": w.num_actors,
|
||||
"mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0),
|
||||
"mailbox_after": w.mailbox_depth,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
CommandResponse::ok(
|
||||
"diff",
|
||||
serde_json::json!({
|
||||
"elapsed_s": elapsed,
|
||||
"actors_before": before.actor_details.len(),
|
||||
"actors_after": after.actor_details.len(),
|
||||
"delta_messages": delta_msgs,
|
||||
"msg_per_sec": delta_msgs as f64 / elapsed,
|
||||
"delta_local_sends": local_after.saturating_sub(local_before),
|
||||
"delta_cross_sends": cross_after.saturating_sub(cross_before),
|
||||
"mailbox_before": mailbox_before,
|
||||
"mailbox_after": mailbox_after,
|
||||
"per_worker": per_worker,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Write Commands ──────────────────────────────────────────────────────────
|
||||
|
||||
pub struct ShutdownCommand;
|
||||
|
||||
impl CommandHandler for ShutdownCommand {
|
||||
fn meta(&self) -> CommandMeta {
|
||||
CommandMeta {
|
||||
name: "shutdown",
|
||||
description: "Signal the runtime to shut down gracefully",
|
||||
usage: "shutdown",
|
||||
is_write: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(
|
||||
&self,
|
||||
_args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
ctx.runtime.shutdown();
|
||||
CommandResponse::ok(
|
||||
"shutdown",
|
||||
serde_json::json!({"status": "shutdown signaled"}),
|
||||
)
|
||||
}
|
||||
}
|
||||
242
crates/command/src/lib.rs
Normal file
242
crates/command/src/lib.rs
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
//! Frontend-agnostic command dispatch for swactor runtimes.
|
||||
//!
|
||||
//! Provides [`CommandRouter`] that maps command names to [`CommandHandler`]
|
||||
//! implementations, with built-in commands for runtime inspection and management.
|
||||
//!
|
||||
//! # Architecture
|
||||
//!
|
||||
//! ```text
|
||||
//! Frontend (REPL, REST, TUI, WebSocket)
|
||||
//! │
|
||||
//! ▼
|
||||
//! CommandRouter::dispatch(CommandRequest, CommandContext)
|
||||
//! │
|
||||
//! ├── built-in handlers (overview, workers, actors, …)
|
||||
//! └── custom handlers (user-registered)
|
||||
//! ```
|
||||
|
||||
pub mod builtins;
|
||||
mod parse;
|
||||
|
||||
pub use parse::{from_query_params, parse_line};
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::runtime::Runtime;
|
||||
use swactor::stats::RuntimeStats;
|
||||
|
||||
// ─── Core Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// A command request.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CommandRequest {
|
||||
pub command: String,
|
||||
pub args: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// A command response. Always JSON-serializable.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CommandResponse {
|
||||
pub ok: bool,
|
||||
pub command: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl CommandResponse {
|
||||
pub fn ok(command: &str, data: impl Serialize) -> Self {
|
||||
Self {
|
||||
ok: true,
|
||||
command: command.to_string(),
|
||||
data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)),
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err(command: &str, msg: impl Into<String>) -> Self {
|
||||
Self {
|
||||
ok: false,
|
||||
command: command.to_string(),
|
||||
data: None,
|
||||
error: Some(msg.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialize to a single JSON line (for REPL/wire protocol).
|
||||
pub fn to_json_line(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|e| {
|
||||
format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Handler Trait ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Metadata about a command, used for help text and validation.
|
||||
pub struct CommandMeta {
|
||||
pub name: &'static str,
|
||||
pub description: &'static str,
|
||||
pub usage: &'static str,
|
||||
pub is_write: bool,
|
||||
}
|
||||
|
||||
/// A command handler. Implementations are stateless — all state
|
||||
/// comes through [`CommandContext`].
|
||||
pub trait CommandHandler: Send + Sync {
|
||||
fn meta(&self) -> CommandMeta;
|
||||
fn handle(
|
||||
&self,
|
||||
args: &HashMap<String, serde_json::Value>,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse;
|
||||
}
|
||||
|
||||
// ─── Stats Enrichment ────────────────────────────────────────────────────────
|
||||
|
||||
/// Enriches [`RuntimeStats`] with per-actor detail.
|
||||
///
|
||||
/// Implement this on your stats collector so command handlers can access
|
||||
/// enriched data without depending on the dashboard crate.
|
||||
pub trait StatsEnricher: Send + Sync {
|
||||
fn enrich(&self, stats: &mut RuntimeStats);
|
||||
}
|
||||
|
||||
// ─── Context ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Context available to command handlers.
|
||||
pub struct CommandContext {
|
||||
pub runtime: Arc<Runtime>,
|
||||
pub enricher: Option<Arc<dyn StatsEnricher>>,
|
||||
}
|
||||
|
||||
impl CommandContext {
|
||||
pub fn new(runtime: Arc<Runtime>) -> Self {
|
||||
Self {
|
||||
runtime,
|
||||
enricher: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_enricher(
|
||||
runtime: Arc<Runtime>,
|
||||
enricher: Arc<dyn StatsEnricher>,
|
||||
) -> Self {
|
||||
Self {
|
||||
runtime,
|
||||
enricher: Some(enricher),
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw runtime stats (no enrichment).
|
||||
pub fn stats(&self) -> RuntimeStats {
|
||||
self.runtime.stats()
|
||||
}
|
||||
|
||||
/// Runtime stats enriched with per-actor detail (if an enricher is set).
|
||||
pub fn enriched_stats(&self) -> RuntimeStats {
|
||||
let mut s = self.runtime.stats();
|
||||
if let Some(e) = &self.enricher {
|
||||
e.enrich(&mut s);
|
||||
}
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Router ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Central command dispatch.
|
||||
pub struct CommandRouter {
|
||||
handlers: HashMap<String, Box<dyn CommandHandler>>,
|
||||
}
|
||||
|
||||
impl CommandRouter {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
handlers: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a router with all built-in commands registered.
|
||||
pub fn with_builtins() -> Self {
|
||||
let mut router = Self::new();
|
||||
router.register(Box::new(builtins::OverviewCommand));
|
||||
router.register(Box::new(builtins::WorkersCommand));
|
||||
router.register(Box::new(builtins::WorkerCommand));
|
||||
router.register(Box::new(builtins::ActorsCommand));
|
||||
router.register(Box::new(builtins::ActorCommand));
|
||||
router.register(Box::new(builtins::HotCommand));
|
||||
router.register(Box::new(builtins::PhasesCommand));
|
||||
router.register(Box::new(builtins::DiffCommand));
|
||||
router.register(Box::new(builtins::ShutdownCommand));
|
||||
router
|
||||
}
|
||||
|
||||
/// Register a custom command handler.
|
||||
pub fn register(&mut self, handler: Box<dyn CommandHandler>) {
|
||||
let name = handler.meta().name.to_string();
|
||||
self.handlers.insert(name, handler);
|
||||
}
|
||||
|
||||
/// Dispatch a command request.
|
||||
///
|
||||
/// The `help` command is handled directly by the router (it needs
|
||||
/// access to all registered handlers).
|
||||
pub fn dispatch(
|
||||
&self,
|
||||
req: &CommandRequest,
|
||||
ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
if req.command == "help" {
|
||||
return self.cmd_help();
|
||||
}
|
||||
match self.handlers.get(&req.command) {
|
||||
Some(handler) => handler.handle(&req.args, ctx),
|
||||
None => CommandResponse::err(
|
||||
&req.command,
|
||||
format!("unknown command `{}` — try `help`", req.command),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_help(&self) -> CommandResponse {
|
||||
let mut commands: Vec<serde_json::Value> = self
|
||||
.handlers
|
||||
.values()
|
||||
.map(|h| {
|
||||
let m = h.meta();
|
||||
serde_json::json!({
|
||||
"name": m.name,
|
||||
"usage": m.usage,
|
||||
"description": m.description,
|
||||
"is_write": m.is_write,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
// Add help itself
|
||||
commands.push(serde_json::json!({
|
||||
"name": "help",
|
||||
"usage": "help",
|
||||
"description": "List all available commands",
|
||||
"is_write": false,
|
||||
}));
|
||||
commands.sort_by(|a, b| {
|
||||
a["name"]
|
||||
.as_str()
|
||||
.unwrap_or("")
|
||||
.cmp(b["name"].as_str().unwrap_or(""))
|
||||
});
|
||||
CommandResponse::ok("help", serde_json::json!({ "commands": commands }))
|
||||
}
|
||||
|
||||
/// List names of all registered commands (sorted).
|
||||
pub fn command_names(&self) -> Vec<&str> {
|
||||
let mut names: Vec<_> = self.handlers.keys().map(|s| s.as_str()).collect();
|
||||
names.push("help");
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
}
|
||||
90
crates/command/src/parse.rs
Normal file
90
crates/command/src/parse.rs
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
//! Input parsers for REPL lines and HTTP query parameters.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::CommandRequest;
|
||||
|
||||
/// Parse a REPL text line into a [`CommandRequest`].
|
||||
///
|
||||
/// Handles `--flag value` pairs and maps positional arguments to
|
||||
/// command-specific named parameters.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```text
|
||||
/// "overview" → { command: "overview", args: {} }
|
||||
/// "worker 3" → { command: "worker", args: { "id": "3" } }
|
||||
/// "actors --sort mailbox" → { command: "actors", args: { "sort": "mailbox" } }
|
||||
/// "hot 5" → { command: "hot", args: { "n": "5" } }
|
||||
/// ```
|
||||
pub fn parse_line(line: &str) -> CommandRequest {
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
if parts.is_empty() {
|
||||
return CommandRequest {
|
||||
command: "help".to_string(),
|
||||
args: HashMap::new(),
|
||||
};
|
||||
}
|
||||
let command = parts[0].to_string();
|
||||
let rest = &parts[1..];
|
||||
|
||||
let mut args = HashMap::new();
|
||||
let mut i = 0;
|
||||
let mut positional = 0;
|
||||
|
||||
while i < rest.len() {
|
||||
if let Some(key) = rest[i].strip_prefix("--") {
|
||||
if i + 1 < rest.len() && !rest[i + 1].starts_with("--") {
|
||||
args.insert(
|
||||
key.to_string(),
|
||||
serde_json::Value::String(rest[i + 1].to_string()),
|
||||
);
|
||||
i += 2;
|
||||
} else {
|
||||
args.insert(key.to_string(), serde_json::Value::Bool(true));
|
||||
i += 1;
|
||||
}
|
||||
} else {
|
||||
let name = positional_arg_name(&command, positional);
|
||||
if !name.is_empty() {
|
||||
args.insert(
|
||||
name.to_string(),
|
||||
serde_json::Value::String(rest[i].to_string()),
|
||||
);
|
||||
}
|
||||
positional += 1;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
CommandRequest { command, args }
|
||||
}
|
||||
|
||||
/// Convert HTTP query parameters to a [`CommandRequest`].
|
||||
///
|
||||
/// The `cmd` parameter becomes the command name; all other parameters
|
||||
/// become string-valued arguments.
|
||||
pub fn from_query_params(params: &HashMap<String, String>) -> CommandRequest {
|
||||
let command = params
|
||||
.get("cmd")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "help".into());
|
||||
let args: HashMap<String, serde_json::Value> = params
|
||||
.iter()
|
||||
.filter(|(k, _)| *k != "cmd")
|
||||
.map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone())))
|
||||
.collect();
|
||||
CommandRequest { command, args }
|
||||
}
|
||||
|
||||
/// Map positional argument index to the named parameter for each command.
|
||||
fn positional_arg_name(command: &str, position: usize) -> &'static str {
|
||||
match (command, position) {
|
||||
("worker", 0) => "id",
|
||||
("actor", 0) => "prefix",
|
||||
("hot", 0) => "n",
|
||||
("phases", 0) => "worker",
|
||||
("diff", 0) => "seconds",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
367
crates/command/tests/command_api.rs
Normal file
367
crates/command/tests/command_api.rs
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
//! Behavioral tests for the swactor-command crate.
|
||||
//!
|
||||
//! Tests exercise the full dispatch path: parse → route → handle → response.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use swactor::actor::ActorInterface;
|
||||
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
|
||||
use swactor_command::{
|
||||
from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter,
|
||||
};
|
||||
|
||||
// ── Test Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
fn single_thread_config() -> RuntimeConfig {
|
||||
RuntimeConfig {
|
||||
num_threads: 1,
|
||||
..RuntimeConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn make_router_and_ctx() -> (CommandRouter, CommandContext) {
|
||||
let rt = Arc::new(Runtime::new(single_thread_config()));
|
||||
let router = CommandRouter::with_builtins();
|
||||
let ctx = CommandContext::new(rt);
|
||||
(router, ctx)
|
||||
}
|
||||
|
||||
fn dispatch_text(router: &CommandRouter, ctx: &CommandContext, line: &str) -> CommandResponse {
|
||||
let req = parse_line(line);
|
||||
let resp = router.dispatch(&req, ctx);
|
||||
// Verify JSON round-trip works
|
||||
let json = resp.to_json_line();
|
||||
serde_json::from_str::<CommandResponse>(&json)
|
||||
.expect("response should be valid JSON")
|
||||
}
|
||||
|
||||
/// A no-op actor for spawning into the runtime.
|
||||
struct DummyActor;
|
||||
#[derive(Clone)]
|
||||
struct DummyMsg;
|
||||
impl ActorInterface for DummyActor {
|
||||
type Incoming = DummyMsg;
|
||||
type Response = ();
|
||||
fn handle(&mut self, _ctx: &Ctx, _msg: DummyMsg) {}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Given a router with builtins,
|
||||
/// when "help" is dispatched,
|
||||
/// then the response lists all registered commands.
|
||||
#[test]
|
||||
fn help_lists_all_registered_commands() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "help");
|
||||
|
||||
assert!(resp.ok, "help should succeed");
|
||||
assert_eq!(resp.command, "help");
|
||||
|
||||
let data = resp.data.unwrap();
|
||||
let commands = data["commands"].as_array().unwrap();
|
||||
|
||||
// Should have all builtins + help itself
|
||||
let names: Vec<&str> = commands
|
||||
.iter()
|
||||
.map(|c| c["name"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(names.contains(&"overview"), "should list overview");
|
||||
assert!(names.contains(&"workers"), "should list workers");
|
||||
assert!(names.contains(&"worker"), "should list worker");
|
||||
assert!(names.contains(&"actors"), "should list actors");
|
||||
assert!(names.contains(&"hot"), "should list hot");
|
||||
assert!(names.contains(&"phases"), "should list phases");
|
||||
assert!(names.contains(&"diff"), "should list diff");
|
||||
assert!(names.contains(&"shutdown"), "should list shutdown");
|
||||
assert!(names.contains(&"help"), "should list help itself");
|
||||
|
||||
// Should be sorted
|
||||
let mut sorted = names.clone();
|
||||
sorted.sort();
|
||||
assert_eq!(names, sorted, "commands should be sorted alphabetically");
|
||||
}
|
||||
|
||||
/// Given a router,
|
||||
/// when an unknown command is dispatched,
|
||||
/// then the response indicates failure with a helpful message.
|
||||
#[test]
|
||||
fn unknown_command_returns_error() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "nonexistent");
|
||||
|
||||
assert!(!resp.ok, "unknown command should fail");
|
||||
assert_eq!(resp.command, "nonexistent");
|
||||
let err = resp.error.unwrap();
|
||||
assert!(
|
||||
err.contains("unknown command") && err.contains("help"),
|
||||
"error should mention 'unknown command' and suggest 'help', got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Given a runtime with no actors,
|
||||
/// when "overview" is dispatched,
|
||||
/// then the response contains expected summary fields with zero counts.
|
||||
#[test]
|
||||
fn overview_returns_summary_fields() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "overview");
|
||||
|
||||
assert!(resp.ok);
|
||||
assert_eq!(resp.command, "overview");
|
||||
|
||||
let data = resp.data.unwrap();
|
||||
assert_eq!(data["workers"], 1, "single-threaded = 1 worker");
|
||||
assert_eq!(data["actors"], 0, "no actors spawned");
|
||||
assert_eq!(data["total_messages_processed"], 0);
|
||||
assert_eq!(data["total_panics"], 0);
|
||||
assert!(data["sends"].is_object(), "sends should be an object");
|
||||
}
|
||||
|
||||
/// Given a runtime with spawned actors,
|
||||
/// when "workers" is dispatched,
|
||||
/// then the response contains per-worker stats.
|
||||
#[test]
|
||||
fn workers_returns_per_worker_info() {
|
||||
let rt = Arc::new(Runtime::new(single_thread_config()));
|
||||
// Spawn some actors
|
||||
rt.spawn(DummyActor).unwrap();
|
||||
rt.spawn(DummyActor).unwrap();
|
||||
rt.tick();
|
||||
|
||||
let router = CommandRouter::with_builtins();
|
||||
let ctx = CommandContext::new(rt);
|
||||
let resp = dispatch_text(&router, &ctx, "workers");
|
||||
|
||||
assert!(resp.ok);
|
||||
let data = resp.data.unwrap();
|
||||
let workers = data.as_array().unwrap();
|
||||
assert_eq!(workers.len(), 1, "single-threaded has 1 worker");
|
||||
assert_eq!(workers[0]["id"], 0);
|
||||
assert_eq!(workers[0]["actors"], 2, "2 actors spawned on worker 0");
|
||||
}
|
||||
|
||||
/// Given "worker 0" with a valid ID,
|
||||
/// when dispatched,
|
||||
/// then the response includes worker detail and tick phase info.
|
||||
#[test]
|
||||
fn worker_command_with_valid_id() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "worker 0");
|
||||
|
||||
assert!(resp.ok);
|
||||
assert_eq!(resp.command, "worker");
|
||||
let data = resp.data.unwrap();
|
||||
assert_eq!(data["id"], 0);
|
||||
assert!(data["tick_phases"].is_object(), "should include phase breakdown");
|
||||
}
|
||||
|
||||
/// Given "worker 99",
|
||||
/// when dispatched,
|
||||
/// then the response is an error (worker not found).
|
||||
#[test]
|
||||
fn worker_command_invalid_id_returns_error() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "worker 99");
|
||||
|
||||
assert!(!resp.ok);
|
||||
assert!(resp.error.unwrap().contains("not found"));
|
||||
}
|
||||
|
||||
/// Given "worker" with no ID,
|
||||
/// when dispatched,
|
||||
/// then the response is a usage error.
|
||||
#[test]
|
||||
fn worker_command_missing_id_returns_usage() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "worker");
|
||||
|
||||
assert!(!resp.ok);
|
||||
assert!(resp.error.unwrap().contains("usage"));
|
||||
}
|
||||
|
||||
/// Given "phases",
|
||||
/// when dispatched,
|
||||
/// then the response includes phase breakdown per worker.
|
||||
#[test]
|
||||
fn phases_command_returns_breakdown() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "phases");
|
||||
|
||||
assert!(resp.ok);
|
||||
let data = resp.data.unwrap();
|
||||
let phases = data.as_array().unwrap();
|
||||
assert_eq!(phases.len(), 1, "single-threaded = 1 worker");
|
||||
assert_eq!(phases[0]["worker_id"], 0);
|
||||
}
|
||||
|
||||
/// Given a runtime, when "shutdown" is dispatched,
|
||||
/// then the response indicates success.
|
||||
#[test]
|
||||
fn shutdown_command_signals_runtime() {
|
||||
let (router, ctx) = make_router_and_ctx();
|
||||
let resp = dispatch_text(&router, &ctx, "shutdown");
|
||||
|
||||
assert!(resp.ok);
|
||||
assert_eq!(resp.command, "shutdown");
|
||||
let data = resp.data.unwrap();
|
||||
assert_eq!(data["status"], "shutdown signaled");
|
||||
}
|
||||
|
||||
// ── REPL Parser Tests ────────────────────────────────────────────────────────
|
||||
|
||||
/// Given a simple command with no args,
|
||||
/// when parsed,
|
||||
/// then the command name is extracted correctly.
|
||||
#[test]
|
||||
fn parse_line_simple_command() {
|
||||
let req = parse_line("overview");
|
||||
assert_eq!(req.command, "overview");
|
||||
assert!(req.args.is_empty());
|
||||
}
|
||||
|
||||
/// Given a command with positional args,
|
||||
/// when parsed,
|
||||
/// then positional args are mapped to named parameters.
|
||||
#[test]
|
||||
fn parse_line_positional_args() {
|
||||
let req = parse_line("worker 3");
|
||||
assert_eq!(req.command, "worker");
|
||||
assert_eq!(req.args["id"], "3");
|
||||
|
||||
let req = parse_line("hot 5");
|
||||
assert_eq!(req.command, "hot");
|
||||
assert_eq!(req.args["n"], "5");
|
||||
|
||||
let req = parse_line("diff 2.5");
|
||||
assert_eq!(req.command, "diff");
|
||||
assert_eq!(req.args["seconds"], "2.5");
|
||||
|
||||
let req = parse_line("actor a1b2");
|
||||
assert_eq!(req.command, "actor");
|
||||
assert_eq!(req.args["prefix"], "a1b2");
|
||||
}
|
||||
|
||||
/// Given a command with --flag value pairs,
|
||||
/// when parsed,
|
||||
/// then flags are mapped to named args.
|
||||
#[test]
|
||||
fn parse_line_flags() {
|
||||
let req = parse_line("actors --sort mailbox --limit 5");
|
||||
assert_eq!(req.command, "actors");
|
||||
assert_eq!(req.args["sort"], "mailbox");
|
||||
assert_eq!(req.args["limit"], "5");
|
||||
}
|
||||
|
||||
/// Given a command with mixed positional and flag args,
|
||||
/// when parsed,
|
||||
/// then both are captured correctly.
|
||||
#[test]
|
||||
fn parse_line_mixed_args() {
|
||||
let req = parse_line("actors --sort worker --worker 2 --limit 10");
|
||||
assert_eq!(req.command, "actors");
|
||||
assert_eq!(req.args["sort"], "worker");
|
||||
assert_eq!(req.args["worker"], "2");
|
||||
assert_eq!(req.args["limit"], "10");
|
||||
}
|
||||
|
||||
/// Given empty input,
|
||||
/// when parsed,
|
||||
/// then default to "help".
|
||||
#[test]
|
||||
fn parse_line_empty_defaults_to_help() {
|
||||
let req = parse_line("");
|
||||
assert_eq!(req.command, "help");
|
||||
}
|
||||
|
||||
// ── REST Adapter Tests ───────────────────────────────────────────────────────
|
||||
|
||||
/// Given query params with cmd and other params,
|
||||
/// when converted,
|
||||
/// then cmd becomes the command and others become args.
|
||||
#[test]
|
||||
fn from_query_params_extracts_cmd() {
|
||||
let mut params = HashMap::new();
|
||||
params.insert("cmd".to_string(), "actor".to_string());
|
||||
params.insert("prefix".to_string(), "a1b2".to_string());
|
||||
|
||||
let req = from_query_params(¶ms);
|
||||
assert_eq!(req.command, "actor");
|
||||
assert_eq!(req.args["prefix"], "a1b2");
|
||||
assert!(!req.args.contains_key("cmd"), "cmd should not be in args");
|
||||
}
|
||||
|
||||
/// Given query params with no cmd,
|
||||
/// when converted,
|
||||
/// then default to "help".
|
||||
#[test]
|
||||
fn from_query_params_defaults_to_help() {
|
||||
let params = HashMap::new();
|
||||
let req = from_query_params(¶ms);
|
||||
assert_eq!(req.command, "help");
|
||||
}
|
||||
|
||||
// ── Custom Handler Test ──────────────────────────────────────────────────────
|
||||
|
||||
/// Given a custom command handler registered on the router,
|
||||
/// when that command is dispatched,
|
||||
/// then the custom handler runs and returns its response.
|
||||
#[test]
|
||||
fn custom_command_handler() {
|
||||
struct PingCommand;
|
||||
impl swactor_command::CommandHandler for PingCommand {
|
||||
fn meta(&self) -> swactor_command::CommandMeta {
|
||||
swactor_command::CommandMeta {
|
||||
name: "ping",
|
||||
description: "Respond with pong",
|
||||
usage: "ping",
|
||||
is_write: false,
|
||||
}
|
||||
}
|
||||
fn handle(
|
||||
&self,
|
||||
_args: &HashMap<String, serde_json::Value>,
|
||||
_ctx: &CommandContext,
|
||||
) -> CommandResponse {
|
||||
CommandResponse::ok("ping", serde_json::json!({"reply": "pong"}))
|
||||
}
|
||||
}
|
||||
|
||||
let rt = Arc::new(Runtime::new(single_thread_config()));
|
||||
let mut router = CommandRouter::with_builtins();
|
||||
router.register(Box::new(PingCommand));
|
||||
let ctx = CommandContext::new(rt);
|
||||
|
||||
let resp = dispatch_text(&router, &ctx, "ping");
|
||||
assert!(resp.ok);
|
||||
assert_eq!(resp.data.unwrap()["reply"], "pong");
|
||||
|
||||
// Should also appear in help
|
||||
let help = dispatch_text(&router, &ctx, "help");
|
||||
let commands = help.data.unwrap()["commands"].as_array().unwrap().clone();
|
||||
let names: Vec<&str> = commands.iter().map(|c| c["name"].as_str().unwrap()).collect();
|
||||
assert!(names.contains(&"ping"), "custom command should appear in help");
|
||||
}
|
||||
|
||||
/// Given a JSON-serialized CommandResponse,
|
||||
/// when deserialized,
|
||||
/// then ok/false fields, data, and error are preserved.
|
||||
#[test]
|
||||
fn response_json_roundtrip() {
|
||||
let ok_resp = CommandResponse::ok("test", serde_json::json!({"key": "value"}));
|
||||
let json = ok_resp.to_json_line();
|
||||
let parsed: CommandResponse = serde_json::from_str(&json).unwrap();
|
||||
assert!(parsed.ok);
|
||||
assert_eq!(parsed.command, "test");
|
||||
assert_eq!(parsed.data.unwrap()["key"], "value");
|
||||
assert!(parsed.error.is_none());
|
||||
|
||||
let err_resp = CommandResponse::err("bad", "something went wrong");
|
||||
let json = err_resp.to_json_line();
|
||||
let parsed: CommandResponse = serde_json::from_str(&json).unwrap();
|
||||
assert!(!parsed.ok);
|
||||
assert_eq!(parsed.command, "bad");
|
||||
assert!(parsed.data.is_none());
|
||||
assert_eq!(parsed.error.unwrap(), "something went wrong");
|
||||
}
|
||||
|
|
@ -7,4 +7,5 @@ pub mod swim;
|
|||
pub mod kademlia;
|
||||
pub mod cache;
|
||||
pub mod node;
|
||||
pub mod registry;
|
||||
pub mod snapshot;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,10 @@ use crate::crypto::Keypair;
|
|||
use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard};
|
||||
use crate::kademlia::repair::{RepairQueue, RepublishTracker};
|
||||
use crate::kademlia::routing_table::RoutingTable;
|
||||
use crate::registry::{
|
||||
pack_combined_piggyback, unpack_combined_piggyback, ClusterRegistry, RegistryConfig,
|
||||
RegistryEvent,
|
||||
};
|
||||
use crate::swim::node::{NodeAction, SwimNode};
|
||||
use crate::swim::probe::SwimConfig;
|
||||
use crate::types::{MemberState, NodeId, NodeRecord};
|
||||
|
|
@ -22,6 +26,7 @@ pub struct DistributedNodeConfig {
|
|||
pub swim: SwimConfig,
|
||||
pub cache_capacity: usize,
|
||||
pub republish_interval: u64,
|
||||
pub registry: RegistryConfig,
|
||||
}
|
||||
|
||||
impl Default for DistributedNodeConfig {
|
||||
|
|
@ -31,6 +36,7 @@ impl Default for DistributedNodeConfig {
|
|||
swim: SwimConfig::default(),
|
||||
cache_capacity: 10_000,
|
||||
republish_interval: 1000,
|
||||
registry: RegistryConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -47,6 +53,7 @@ pub struct DistributedNode {
|
|||
cache: LocationCache,
|
||||
repair_queue: RepairQueue,
|
||||
republish: RepublishTracker,
|
||||
registry: ClusterRegistry,
|
||||
tick_count: u64,
|
||||
}
|
||||
|
||||
|
|
@ -67,6 +74,7 @@ impl DistributedNode {
|
|||
cache: LocationCache::new(config.cache_capacity),
|
||||
repair_queue: RepairQueue::new(),
|
||||
republish: RepublishTracker::new(config.republish_interval),
|
||||
registry: ClusterRegistry::new(config.registry),
|
||||
tick_count: 0,
|
||||
keypair,
|
||||
}
|
||||
|
|
@ -149,23 +157,32 @@ impl DistributedNode {
|
|||
// re-sign and re-STORE these entries.
|
||||
}
|
||||
|
||||
actions
|
||||
// Registry GC
|
||||
self.registry.gc_tick();
|
||||
|
||||
// Wrap outgoing piggyback with registry entries
|
||||
self.inject_registry_piggyback(actions)
|
||||
}
|
||||
|
||||
// ─── SWIM message handling (delegate to SwimNode) ───────────────────
|
||||
|
||||
pub fn handle_ping(&mut self, from: NodeId, from_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
|
||||
let actions = self.swim.handle_ping(from, from_addr, sequence, piggyback);
|
||||
let membership_bytes = self.extract_registry_piggyback(piggyback);
|
||||
let actions = self.swim.handle_ping(from, from_addr, sequence, &membership_bytes);
|
||||
self.maybe_update_routing_table(from, from_addr);
|
||||
actions
|
||||
self.inject_registry_piggyback(actions)
|
||||
}
|
||||
|
||||
pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
|
||||
self.swim.handle_ack(from, sequence, piggyback)
|
||||
let membership_bytes = self.extract_registry_piggyback(piggyback);
|
||||
let actions = self.swim.handle_ack(from, sequence, &membership_bytes);
|
||||
self.inject_registry_piggyback(actions)
|
||||
}
|
||||
|
||||
pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, target_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec<NodeAction> {
|
||||
self.swim.handle_ping_req(from, target, target_addr, sequence, piggyback)
|
||||
let membership_bytes = self.extract_registry_piggyback(piggyback);
|
||||
let actions = self.swim.handle_ping_req(from, target, target_addr, sequence, &membership_bytes);
|
||||
self.inject_registry_piggyback(actions)
|
||||
}
|
||||
|
||||
pub fn handle_join_request(&mut self, from: NodeId, from_addr: SocketAddr) -> Vec<NodeAction> {
|
||||
|
|
@ -233,6 +250,33 @@ impl DistributedNode {
|
|||
self.cache.invalidate(actor_addr);
|
||||
}
|
||||
|
||||
// ─── Registry (name → actor mapping) ──────────────────────────────
|
||||
|
||||
/// Register a human-readable name for an actor on this node.
|
||||
pub fn register_name(&mut self, name: String, actor_addr: ActorAddress) {
|
||||
self.registry.register(name, actor_addr, self.node_id(), self.cluster_size());
|
||||
}
|
||||
|
||||
/// Unregister a name (creates a tombstone).
|
||||
pub fn unregister_name(&mut self, name: &str) {
|
||||
self.registry.unregister(name, self.node_id(), self.cluster_size());
|
||||
}
|
||||
|
||||
/// Resolve a name to its current (ActorAddress, NodeId).
|
||||
pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
|
||||
self.registry.resolve(name)
|
||||
}
|
||||
|
||||
/// Drain registry events (Registered / Unregistered).
|
||||
pub fn registry_events(&mut self) -> Vec<RegistryEvent> {
|
||||
self.registry.drain_events()
|
||||
}
|
||||
|
||||
/// Read-only access to the registry.
|
||||
pub fn registry(&self) -> &ClusterRegistry {
|
||||
&self.registry
|
||||
}
|
||||
|
||||
// ─── Accessors ──────────────────────────────────────────────────────
|
||||
|
||||
pub fn routing_table(&self) -> &RoutingTable {
|
||||
|
|
@ -277,12 +321,51 @@ impl DistributedNode {
|
|||
self.routing_table.remove(&node_id);
|
||||
self.cache.invalidate_node(&node_id);
|
||||
self.repair_queue.on_node_death(&node_id, &mut self.directory);
|
||||
self.registry.tombstone_node(node_id, self.cluster_size());
|
||||
}
|
||||
MemberState::Suspect => {
|
||||
// Keep in routing table but could downprioritize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cluster_size(&self) -> usize {
|
||||
self.swim.members().alive_count() + 1 // +1 for self
|
||||
}
|
||||
|
||||
/// Post-process outgoing actions: wrap each piggyback with registry entries.
|
||||
fn inject_registry_piggyback(&mut self, actions: Vec<NodeAction>) -> Vec<NodeAction> {
|
||||
actions
|
||||
.into_iter()
|
||||
.map(|action| match action {
|
||||
NodeAction::SendPing { to, to_addr, sequence, piggyback } => {
|
||||
let registry_entries = self.registry.take_pending(8);
|
||||
let combined = pack_combined_piggyback(piggyback, registry_entries);
|
||||
NodeAction::SendPing { to, to_addr, sequence, piggyback: combined }
|
||||
}
|
||||
NodeAction::SendAck { to, to_addr, sequence, piggyback } => {
|
||||
let registry_entries = self.registry.take_pending(8);
|
||||
let combined = pack_combined_piggyback(piggyback, registry_entries);
|
||||
NodeAction::SendAck { to, to_addr, sequence, piggyback: combined }
|
||||
}
|
||||
NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback } => {
|
||||
let registry_entries = self.registry.take_pending(8);
|
||||
let combined = pack_combined_piggyback(piggyback, registry_entries);
|
||||
NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback: combined }
|
||||
}
|
||||
other => other,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Extract registry entries from incoming piggyback, merge them, return membership-only bytes.
|
||||
fn extract_registry_piggyback(&mut self, bytes: &[u8]) -> Vec<u8> {
|
||||
let (membership_bytes, registry_entries) = unpack_combined_piggyback(bytes);
|
||||
if !registry_entries.is_empty() {
|
||||
self.registry.merge_batch(registry_entries, self.cluster_size());
|
||||
}
|
||||
membership_bytes
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of resolving an actor's location.
|
||||
|
|
|
|||
374
crates/distribution/src/registry.rs
Normal file
374
crates/distribution/src/registry.rs
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
//! Cluster Registry — gossip-propagated naming via LWW-Register CRDT.
|
||||
//!
|
||||
//! Maps human-readable names to `(ActorAddress, NodeId)` pairs, propagated
|
||||
//! through SWIM gossip piggyback. Uses last-writer-wins semantics with
|
||||
//! tie-breaking on (timestamp, generation, node_id).
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::actor::ActorAddress;
|
||||
|
||||
use crate::types::NodeId;
|
||||
|
||||
// ─── Configuration ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Configuration for the cluster registry.
|
||||
pub struct RegistryConfig {
|
||||
/// Maximum number of events to buffer before dropping old ones.
|
||||
pub max_events: usize,
|
||||
/// How long (in ticks) a tombstone is retained before GC.
|
||||
pub tombstone_ttl: u64,
|
||||
/// How often (in ticks) to run garbage collection.
|
||||
pub gc_interval: u64,
|
||||
/// Dissemination multiplier (Λ) — same role as in SWIM dissemination.
|
||||
pub dissemination_lambda: usize,
|
||||
}
|
||||
|
||||
impl Default for RegistryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_events: 256,
|
||||
tombstone_ttl: 3600,
|
||||
gc_interval: 1000,
|
||||
dissemination_lambda: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Wire types ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// A single registry entry — the unit of replication.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct RegistryEntry {
|
||||
pub name: String,
|
||||
pub actor_addr: ActorAddress,
|
||||
pub node_id: NodeId,
|
||||
/// Logical timestamp (monotonically increasing per-registry).
|
||||
pub timestamp: u64,
|
||||
/// Generation counter for the same name (disambiguates re-registrations).
|
||||
pub generation: u64,
|
||||
/// If true, this entry is a tombstone (name was unregistered).
|
||||
pub tombstone: bool,
|
||||
}
|
||||
|
||||
/// Combined piggyback payload: membership bytes + registry entries.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PiggybackPayload {
|
||||
/// Raw SWIM membership piggyback bytes (opaque to registry).
|
||||
pub membership: Vec<u8>,
|
||||
/// Registry entries to disseminate.
|
||||
pub registry: Vec<RegistryEntry>,
|
||||
}
|
||||
|
||||
// ─── Events ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Events emitted when the registry changes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RegistryEvent {
|
||||
Registered {
|
||||
name: String,
|
||||
actor_addr: ActorAddress,
|
||||
node_id: NodeId,
|
||||
},
|
||||
Unregistered {
|
||||
name: String,
|
||||
previous_addr: ActorAddress,
|
||||
},
|
||||
}
|
||||
|
||||
// ─── Dissemination entry ────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct DisseminationEntry {
|
||||
entry: RegistryEntry,
|
||||
remaining: usize,
|
||||
}
|
||||
|
||||
// ─── ClusterRegistry ────────────────────────────────────────────────────────
|
||||
|
||||
/// CRDT-based cluster registry with LWW semantics and gossip dissemination.
|
||||
pub struct ClusterRegistry {
|
||||
/// Current state: name → latest entry.
|
||||
entries: HashMap<String, RegistryEntry>,
|
||||
/// Pending entries to disseminate via piggyback.
|
||||
dissemination: Vec<DisseminationEntry>,
|
||||
/// Monotonic logical clock for this node's writes.
|
||||
clock: u64,
|
||||
/// Buffered events for consumers.
|
||||
events: VecDeque<RegistryEvent>,
|
||||
config: RegistryConfig,
|
||||
tick_count: u64,
|
||||
}
|
||||
|
||||
impl ClusterRegistry {
|
||||
pub fn new(config: RegistryConfig) -> Self {
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
dissemination: Vec::new(),
|
||||
clock: 0,
|
||||
events: VecDeque::new(),
|
||||
config,
|
||||
tick_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a name → actor binding from the local node.
|
||||
pub fn register(&mut self, name: String, actor_addr: ActorAddress, node_id: NodeId, cluster_size: usize) {
|
||||
self.clock += 1;
|
||||
let generation = self.next_generation(&name);
|
||||
let entry = RegistryEntry {
|
||||
name,
|
||||
actor_addr,
|
||||
node_id,
|
||||
timestamp: self.clock,
|
||||
generation,
|
||||
tombstone: false,
|
||||
};
|
||||
self.merge_and_enqueue(entry, cluster_size);
|
||||
}
|
||||
|
||||
/// Unregister a name (create a tombstone).
|
||||
pub fn unregister(&mut self, name: &str, node_id: NodeId, cluster_size: usize) {
|
||||
self.clock += 1;
|
||||
let generation = self.next_generation(name);
|
||||
// Use the existing actor_addr if present, otherwise a zero address.
|
||||
let actor_addr = self.entries
|
||||
.get(name)
|
||||
.map(|e| e.actor_addr)
|
||||
.unwrap_or(ActorAddress([0; 32]));
|
||||
let entry = RegistryEntry {
|
||||
name: name.to_string(),
|
||||
actor_addr,
|
||||
node_id,
|
||||
timestamp: self.clock,
|
||||
generation,
|
||||
tombstone: true,
|
||||
};
|
||||
self.merge_and_enqueue(entry, cluster_size);
|
||||
}
|
||||
|
||||
/// Resolve a name to its current (ActorAddress, NodeId), or None if
|
||||
/// not registered or tombstoned.
|
||||
pub fn resolve(&self, name: &str) -> Option<(ActorAddress, NodeId)> {
|
||||
self.entries.get(name).and_then(|e| {
|
||||
if e.tombstone {
|
||||
None
|
||||
} else {
|
||||
Some((e.actor_addr, e.node_id))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Merge a single remote entry. Returns true if state changed.
|
||||
pub fn merge(&mut self, remote: RegistryEntry) -> bool {
|
||||
if let Some(existing) = self.entries.get(&remote.name) {
|
||||
if !lww_wins(&remote, existing) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
let changed = match self.entries.get(&remote.name) {
|
||||
Some(existing) => existing != &remote,
|
||||
None => true,
|
||||
};
|
||||
|
||||
if changed {
|
||||
self.emit_event(&remote);
|
||||
// Advance clock to stay ahead of remote timestamps.
|
||||
if remote.timestamp >= self.clock {
|
||||
self.clock = remote.timestamp + 1;
|
||||
}
|
||||
}
|
||||
|
||||
self.entries.insert(remote.name.clone(), remote);
|
||||
changed
|
||||
}
|
||||
|
||||
/// Merge a batch of entries received from gossip.
|
||||
/// Changed entries are re-enqueued for further dissemination.
|
||||
pub fn merge_batch(&mut self, entries: Vec<RegistryEntry>, cluster_size: usize) {
|
||||
for entry in entries {
|
||||
if self.merge(entry.clone()) {
|
||||
self.enqueue(entry, cluster_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Take pending entries for piggyback, up to `max_count`.
|
||||
pub fn take_pending(&mut self, max_count: usize) -> Vec<RegistryEntry> {
|
||||
let count = max_count.min(self.dissemination.len());
|
||||
let mut result = Vec::with_capacity(count);
|
||||
|
||||
for entry in self.dissemination.iter_mut().take(count) {
|
||||
result.push(entry.entry.clone());
|
||||
entry.remaining = entry.remaining.saturating_sub(1);
|
||||
}
|
||||
|
||||
// Evict exhausted entries.
|
||||
self.dissemination.retain(|e| e.remaining > 0);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Tombstone all entries owned by a dead node.
|
||||
pub fn tombstone_node(&mut self, dead_node_id: NodeId, cluster_size: usize) {
|
||||
let owned: Vec<String> = self.entries
|
||||
.iter()
|
||||
.filter(|(_, e)| e.node_id == dead_node_id && !e.tombstone)
|
||||
.map(|(name, _)| name.clone())
|
||||
.collect();
|
||||
|
||||
for name in owned {
|
||||
self.clock += 1;
|
||||
let generation = self.next_generation(&name);
|
||||
let actor_addr = self.entries[&name].actor_addr;
|
||||
let entry = RegistryEntry {
|
||||
name,
|
||||
actor_addr,
|
||||
node_id: dead_node_id,
|
||||
timestamp: self.clock,
|
||||
generation,
|
||||
tombstone: true,
|
||||
};
|
||||
self.merge_and_enqueue(entry, cluster_size);
|
||||
}
|
||||
}
|
||||
|
||||
/// Periodic GC: remove tombstones past TTL with exhausted dissemination budgets.
|
||||
pub fn gc_tick(&mut self) {
|
||||
self.tick_count += 1;
|
||||
if self.tick_count % self.config.gc_interval != 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let ttl = self.config.tombstone_ttl;
|
||||
let clock = self.clock;
|
||||
// Names still being disseminated — don't GC those.
|
||||
let pending_names: std::collections::HashSet<String> = self.dissemination
|
||||
.iter()
|
||||
.map(|e| e.entry.name.clone())
|
||||
.collect();
|
||||
|
||||
self.entries.retain(|name, entry| {
|
||||
if entry.tombstone && !pending_names.contains(name) {
|
||||
// Remove if old enough.
|
||||
let age = clock.saturating_sub(entry.timestamp);
|
||||
age < ttl
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Drain buffered events.
|
||||
pub fn drain_events(&mut self) -> Vec<RegistryEvent> {
|
||||
self.events.drain(..).collect()
|
||||
}
|
||||
|
||||
/// Number of registry entries (including tombstones).
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Number of tombstones.
|
||||
pub fn tombstone_count(&self) -> usize {
|
||||
self.entries.values().filter(|e| e.tombstone).count()
|
||||
}
|
||||
|
||||
/// Iterate all entries (for snapshot).
|
||||
pub fn entries(&self) -> impl Iterator<Item = &RegistryEntry> {
|
||||
self.entries.values()
|
||||
}
|
||||
|
||||
// ─── Internal ───────────────────────────────────────────────────────
|
||||
|
||||
fn next_generation(&self, name: &str) -> u64 {
|
||||
self.entries
|
||||
.get(name)
|
||||
.map(|e| e.generation + 1)
|
||||
.unwrap_or(1)
|
||||
}
|
||||
|
||||
fn transmit_budget(&self, cluster_size: usize) -> usize {
|
||||
let n = cluster_size.max(2) as f64;
|
||||
let log_n = n.log2().ceil() as usize;
|
||||
self.config.dissemination_lambda * log_n.max(1)
|
||||
}
|
||||
|
||||
fn enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) {
|
||||
let budget = self.transmit_budget(cluster_size);
|
||||
|
||||
// Replace existing entry for same name if present.
|
||||
if let Some(existing) = self.dissemination.iter_mut().find(|e| e.entry.name == entry.name) {
|
||||
existing.entry = entry;
|
||||
existing.remaining = budget;
|
||||
return;
|
||||
}
|
||||
|
||||
self.dissemination.push(DisseminationEntry {
|
||||
entry,
|
||||
remaining: budget,
|
||||
});
|
||||
}
|
||||
|
||||
fn merge_and_enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) {
|
||||
let merged = self.merge(entry.clone());
|
||||
if merged {
|
||||
self.enqueue(entry, cluster_size);
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_event(&mut self, entry: &RegistryEntry) {
|
||||
let event = if entry.tombstone {
|
||||
RegistryEvent::Unregistered {
|
||||
name: entry.name.clone(),
|
||||
previous_addr: entry.actor_addr,
|
||||
}
|
||||
} else {
|
||||
RegistryEvent::Registered {
|
||||
name: entry.name.clone(),
|
||||
actor_addr: entry.actor_addr,
|
||||
node_id: entry.node_id,
|
||||
}
|
||||
};
|
||||
self.events.push_back(event);
|
||||
while self.events.len() > self.config.max_events {
|
||||
self.events.pop_front();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── LWW conflict resolution ───────────────────────────────────────────────
|
||||
|
||||
/// Returns true if `incoming` wins over `existing` under LWW rules:
|
||||
/// higher timestamp > higher generation > higher node_id (byte-level).
|
||||
fn lww_wins(incoming: &RegistryEntry, existing: &RegistryEntry) -> bool {
|
||||
if incoming.timestamp != existing.timestamp {
|
||||
return incoming.timestamp > existing.timestamp;
|
||||
}
|
||||
if incoming.generation != existing.generation {
|
||||
return incoming.generation > existing.generation;
|
||||
}
|
||||
incoming.node_id.0 > existing.node_id.0
|
||||
}
|
||||
|
||||
// ─── Piggyback pack/unpack ──────────────────────────────────────────────────
|
||||
|
||||
/// Combine membership piggyback bytes and registry entries into a single payload.
|
||||
pub fn pack_combined_piggyback(membership: Vec<u8>, registry: Vec<RegistryEntry>) -> Vec<u8> {
|
||||
let payload = PiggybackPayload { membership, registry };
|
||||
serde_json::to_vec(&payload).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Split a combined piggyback payload into membership bytes and registry entries.
|
||||
/// If deserialization fails, treats the entire blob as membership bytes (backwards compat).
|
||||
pub fn unpack_combined_piggyback(bytes: &[u8]) -> (Vec<u8>, Vec<RegistryEntry>) {
|
||||
if bytes.is_empty() {
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
match serde_json::from_slice::<PiggybackPayload>(bytes) {
|
||||
Ok(payload) => (payload.membership, payload.registry),
|
||||
Err(_) => (bytes.to_vec(), Vec::new()),
|
||||
}
|
||||
}
|
||||
|
|
@ -33,6 +33,15 @@ pub struct CacheEntryInfo {
|
|||
pub node_id: String,
|
||||
}
|
||||
|
||||
/// Snapshot of a single registry entry.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RegistryEntryInfo {
|
||||
pub name: String,
|
||||
pub actor_addr: String,
|
||||
pub node_id: String,
|
||||
pub tombstone: bool,
|
||||
}
|
||||
|
||||
/// Complete snapshot of a `DistributedNode`'s observable state.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DistributionNodeSnapshot {
|
||||
|
|
@ -71,6 +80,14 @@ pub struct DistributionNodeSnapshot {
|
|||
/// Number of entries pending re-replication.
|
||||
pub repair_queue_size: usize,
|
||||
|
||||
// ─── Registry ────────────────────────────────────────────────────
|
||||
/// Number of entries in the cluster registry (including tombstones).
|
||||
pub registry_size: usize,
|
||||
/// Number of tombstoned entries.
|
||||
pub registry_tombstones: usize,
|
||||
/// All registry entries.
|
||||
pub registry_entries: Vec<RegistryEntryInfo>,
|
||||
|
||||
// ─── Gossip pairs ────────────────────────────────────────────────
|
||||
/// Recent SWIM probe targets (most recent last).
|
||||
pub recent_probe_targets: Vec<String>,
|
||||
|
|
@ -136,6 +153,17 @@ impl DistributedNode {
|
|||
.map(|id| node_id_hex(id))
|
||||
.collect();
|
||||
|
||||
let registry = self.registry();
|
||||
let registry_entries: Vec<RegistryEntryInfo> = registry
|
||||
.entries()
|
||||
.map(|e| RegistryEntryInfo {
|
||||
name: e.name.clone(),
|
||||
actor_addr: format!("{}", e.actor_addr),
|
||||
node_id: node_id_hex(&e.node_id),
|
||||
tombstone: e.tombstone,
|
||||
})
|
||||
.collect();
|
||||
|
||||
DistributionNodeSnapshot {
|
||||
node_id: node_id_hex(&self.node_id()),
|
||||
listen_addr: addr_str(&self.listen_addr()),
|
||||
|
|
@ -150,6 +178,9 @@ impl DistributedNode {
|
|||
cache_entries,
|
||||
directory_entry_count: self.directory().entry_count(),
|
||||
repair_queue_size: self.repair_queue_len(),
|
||||
registry_size: registry.len(),
|
||||
registry_tombstones: registry.tombstone_count(),
|
||||
registry_entries,
|
||||
recent_probe_targets: recent_targets,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use swactor::actor::ActorAddress;
|
|||
use distribution::crypto::Keypair;
|
||||
use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult};
|
||||
use distribution::swim::node::NodeAction;
|
||||
use distribution::registry::RegistryConfig;
|
||||
use distribution::swim::probe::SwimConfig;
|
||||
use distribution::types::NodeId;
|
||||
|
||||
|
|
@ -23,6 +24,7 @@ fn test_config(addr: &str) -> DistributedNodeConfig {
|
|||
},
|
||||
cache_capacity: 100,
|
||||
republish_interval: 50,
|
||||
registry: RegistryConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
510
crates/distribution/tests/registry.rs
Normal file
510
crates/distribution/tests/registry.rs
Normal file
|
|
@ -0,0 +1,510 @@
|
|||
//! Behavioral tests for the cluster registry.
|
||||
//!
|
||||
//! Tests gossip-propagated naming via LWW-Register CRDT, using the same
|
||||
//! `deliver_actions` + `test_config` pattern from `node_integration.rs`.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use swactor::actor::ActorAddress;
|
||||
use distribution::node::{DistributedNode, DistributedNodeConfig};
|
||||
use distribution::registry::{ClusterRegistry, RegistryConfig, RegistryEntry, RegistryEvent};
|
||||
use distribution::swim::node::NodeAction;
|
||||
use distribution::swim::probe::SwimConfig;
|
||||
use distribution::types::NodeId;
|
||||
|
||||
fn test_config(addr: &str) -> DistributedNodeConfig {
|
||||
DistributedNodeConfig {
|
||||
listen_addr: addr.parse().unwrap(),
|
||||
swim: SwimConfig {
|
||||
probe_interval: 1,
|
||||
probe_timeout: 3,
|
||||
indirect_probes: 1,
|
||||
suspicion_timeout: 5,
|
||||
},
|
||||
cache_capacity: 100,
|
||||
republish_interval: 50,
|
||||
registry: RegistryConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Simulate a network round: deliver actions from `sender` to the appropriate
|
||||
/// `receiver` node. Returns any actions generated by the receiver.
|
||||
fn deliver_actions(
|
||||
actions: &[NodeAction],
|
||||
sender_id: NodeId,
|
||||
sender_addr: SocketAddr,
|
||||
nodes: &mut [(NodeId, SocketAddr, &mut DistributedNode)],
|
||||
) -> Vec<NodeAction> {
|
||||
let mut responses = Vec::new();
|
||||
for action in actions {
|
||||
match action {
|
||||
NodeAction::SendPing { to, sequence, piggyback, .. } => {
|
||||
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
|
||||
responses.extend(node.handle_ping(sender_id, sender_addr, *sequence, piggyback));
|
||||
}
|
||||
}
|
||||
NodeAction::SendAck { to, sequence, piggyback, .. } => {
|
||||
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
|
||||
responses.extend(node.handle_ack(sender_id, *sequence, piggyback));
|
||||
}
|
||||
}
|
||||
NodeAction::SendJoinRequest { to_addr } => {
|
||||
if let Some((_, _, node)) = nodes.iter_mut().find(|(_, addr, _)| addr == to_addr) {
|
||||
responses.extend(node.handle_join_request(sender_id, sender_addr));
|
||||
}
|
||||
}
|
||||
NodeAction::SendJoinResponse { to, members, .. } => {
|
||||
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) {
|
||||
responses.extend(node.handle_join_response(members.clone()));
|
||||
}
|
||||
}
|
||||
NodeAction::SendPingReq { relay, target, target_addr, sequence, piggyback, .. } => {
|
||||
if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == relay) {
|
||||
responses.extend(node.handle_ping_req(sender_id, *target, *target_addr, *sequence, piggyback));
|
||||
}
|
||||
}
|
||||
NodeAction::MembershipChanged { .. } => {}
|
||||
}
|
||||
}
|
||||
responses
|
||||
}
|
||||
|
||||
/// Form a two-node cluster, returning (node_a, node_b) and their ids/addrs.
|
||||
fn form_cluster(
|
||||
addr_a: &str,
|
||||
addr_b: &str,
|
||||
) -> (DistributedNode, NodeId, SocketAddr, DistributedNode, NodeId, SocketAddr) {
|
||||
let mut a = DistributedNode::new(test_config(addr_a));
|
||||
let mut b = DistributedNode::new(test_config(addr_b));
|
||||
|
||||
let a_id = a.node_id();
|
||||
let a_addr = a.listen_addr();
|
||||
let b_id = b.node_id();
|
||||
let b_addr = b.listen_addr();
|
||||
|
||||
let actions = b.join(&[a_addr]);
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a)];
|
||||
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
|
||||
let mut nodes = vec![(b_id, b_addr, &mut b)];
|
||||
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
|
||||
|
||||
(a, a_id, a_addr, b, b_id, b_addr)
|
||||
}
|
||||
|
||||
/// Run several gossip rounds between two nodes.
|
||||
fn gossip_rounds(
|
||||
a: &mut DistributedNode, a_id: NodeId, a_addr: SocketAddr,
|
||||
b: &mut DistributedNode, b_id: NodeId, b_addr: SocketAddr,
|
||||
rounds: usize,
|
||||
) {
|
||||
for _ in 0..rounds {
|
||||
let actions_a = a.tick();
|
||||
let mut nodes = vec![(b_id, b_addr, &mut *b)];
|
||||
let responses = deliver_actions(&actions_a, a_id, a_addr, &mut nodes);
|
||||
let mut nodes = vec![(a_id, a_addr, &mut *a)];
|
||||
let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes);
|
||||
|
||||
let actions_b = b.tick();
|
||||
let mut nodes = vec![(a_id, a_addr, &mut *a)];
|
||||
let responses = deliver_actions(&actions_b, b_id, b_addr, &mut nodes);
|
||||
let mut nodes = vec![(b_id, b_addr, &mut *b)];
|
||||
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Test 1: register and resolve ───────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn register_and_resolve() {
|
||||
let mut node = DistributedNode::new(test_config("127.0.0.1:10001"));
|
||||
let actor = ActorAddress::new_random();
|
||||
let node_id = node.node_id();
|
||||
|
||||
node.register_name("my-actor".into(), actor);
|
||||
|
||||
let result = node.resolve_name("my-actor");
|
||||
assert_eq!(result, Some((actor, node_id)));
|
||||
}
|
||||
|
||||
// ─── Test 2: unregistered name returns None ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn unregistered_name_returns_none() {
|
||||
let node = DistributedNode::new(test_config("127.0.0.1:10002"));
|
||||
assert_eq!(node.resolve_name("nonexistent"), None);
|
||||
}
|
||||
|
||||
// ─── Test 3: unregister tombstones name ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn unregister_tombstones_name() {
|
||||
let mut node = DistributedNode::new(test_config("127.0.0.1:10003"));
|
||||
let actor = ActorAddress::new_random();
|
||||
|
||||
node.register_name("service".into(), actor);
|
||||
assert!(node.resolve_name("service").is_some());
|
||||
|
||||
node.unregister_name("service");
|
||||
assert_eq!(node.resolve_name("service"), None);
|
||||
}
|
||||
|
||||
// ─── Test 4: re-registration updates binding ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn re_registration_updates_binding() {
|
||||
let mut node = DistributedNode::new(test_config("127.0.0.1:10004"));
|
||||
let actor_a = ActorAddress::new_random();
|
||||
let actor_b = ActorAddress::new_random();
|
||||
let node_id = node.node_id();
|
||||
|
||||
node.register_name("foo".into(), actor_a);
|
||||
assert_eq!(node.resolve_name("foo"), Some((actor_a, node_id)));
|
||||
|
||||
node.register_name("foo".into(), actor_b);
|
||||
assert_eq!(node.resolve_name("foo"), Some((actor_b, node_id)));
|
||||
}
|
||||
|
||||
// ─── Test 5: LWW conflict — higher timestamp wins ──────────────────────────
|
||||
|
||||
#[test]
|
||||
fn lww_conflict_higher_timestamp_wins() {
|
||||
let mut reg = ClusterRegistry::new(RegistryConfig::default());
|
||||
let addr_old = ActorAddress::new_random();
|
||||
let addr_new = ActorAddress::new_random();
|
||||
let node_id = NodeId([1; 32]);
|
||||
|
||||
let old_entry = RegistryEntry {
|
||||
name: "svc".into(),
|
||||
actor_addr: addr_old,
|
||||
node_id,
|
||||
timestamp: 1,
|
||||
generation: 1,
|
||||
tombstone: false,
|
||||
};
|
||||
let new_entry = RegistryEntry {
|
||||
name: "svc".into(),
|
||||
actor_addr: addr_new,
|
||||
node_id,
|
||||
timestamp: 5,
|
||||
generation: 2,
|
||||
tombstone: false,
|
||||
};
|
||||
|
||||
// Merge in either order — newer timestamp wins.
|
||||
reg.merge(new_entry.clone());
|
||||
reg.merge(old_entry.clone());
|
||||
|
||||
assert_eq!(reg.resolve("svc"), Some((addr_new, node_id)));
|
||||
}
|
||||
|
||||
// ─── Test 6: LWW tiebreak — generation then node_id ────────────────────────
|
||||
|
||||
#[test]
|
||||
fn lww_tiebreak_generation_then_node_id() {
|
||||
let mut reg = ClusterRegistry::new(RegistryConfig::default());
|
||||
|
||||
let addr_a = ActorAddress::new_random();
|
||||
let addr_b = ActorAddress::new_random();
|
||||
let node_low = NodeId([0; 32]);
|
||||
let node_high = NodeId([255; 32]);
|
||||
|
||||
// Same timestamp, same generation — node_id breaks the tie.
|
||||
let entry_low = RegistryEntry {
|
||||
name: "x".into(),
|
||||
actor_addr: addr_a,
|
||||
node_id: node_low,
|
||||
timestamp: 10,
|
||||
generation: 1,
|
||||
tombstone: false,
|
||||
};
|
||||
let entry_high = RegistryEntry {
|
||||
name: "x".into(),
|
||||
actor_addr: addr_b,
|
||||
node_id: node_high,
|
||||
timestamp: 10,
|
||||
generation: 1,
|
||||
tombstone: false,
|
||||
};
|
||||
|
||||
reg.merge(entry_low);
|
||||
reg.merge(entry_high);
|
||||
|
||||
// Higher node_id wins.
|
||||
assert_eq!(reg.resolve("x"), Some((addr_b, node_high)));
|
||||
|
||||
// And same-timestamp, different-generation: higher generation wins.
|
||||
let mut reg2 = ClusterRegistry::new(RegistryConfig::default());
|
||||
let entry_gen1 = RegistryEntry {
|
||||
name: "y".into(),
|
||||
actor_addr: addr_a,
|
||||
node_id: node_low,
|
||||
timestamp: 10,
|
||||
generation: 1,
|
||||
tombstone: false,
|
||||
};
|
||||
let entry_gen2 = RegistryEntry {
|
||||
name: "y".into(),
|
||||
actor_addr: addr_b,
|
||||
node_id: node_low,
|
||||
timestamp: 10,
|
||||
generation: 2,
|
||||
tombstone: false,
|
||||
};
|
||||
reg2.merge(entry_gen1);
|
||||
reg2.merge(entry_gen2);
|
||||
assert_eq!(reg2.resolve("y"), Some((addr_b, node_low)));
|
||||
}
|
||||
|
||||
// ─── Test 7: gossip propagates registration ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn gossip_propagates_registration() {
|
||||
let (mut a, a_id, a_addr, mut b, b_id, b_addr) =
|
||||
form_cluster("127.0.0.1:10010", "127.0.0.1:10011");
|
||||
|
||||
let actor = ActorAddress::new_random();
|
||||
a.register_name("greeter".into(), actor);
|
||||
|
||||
// B doesn't know about "greeter" yet.
|
||||
assert_eq!(b.resolve_name("greeter"), None);
|
||||
|
||||
// Run gossip rounds — registry entries piggyback on SWIM messages.
|
||||
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
|
||||
|
||||
// Now B should resolve "greeter" to A's actor.
|
||||
assert_eq!(b.resolve_name("greeter"), Some((actor, a_id)));
|
||||
}
|
||||
|
||||
// ─── Test 8: tombstone propagation via gossip ───────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tombstone_propagation_via_gossip() {
|
||||
let (mut a, a_id, a_addr, mut b, b_id, b_addr) =
|
||||
form_cluster("127.0.0.1:10020", "127.0.0.1:10021");
|
||||
|
||||
let actor = ActorAddress::new_random();
|
||||
a.register_name("ephemeral".into(), actor);
|
||||
|
||||
// Propagate the registration.
|
||||
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
|
||||
assert_eq!(b.resolve_name("ephemeral"), Some((actor, a_id)));
|
||||
|
||||
// Now unregister on A.
|
||||
a.unregister_name("ephemeral");
|
||||
|
||||
// Propagate the tombstone.
|
||||
gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5);
|
||||
|
||||
assert_eq!(b.resolve_name("ephemeral"), None);
|
||||
}
|
||||
|
||||
// ─── Test 9: node death tombstones entries ──────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn node_death_tombstones_entries() {
|
||||
// Set up a 3-node cluster: A, B, C
|
||||
let mut a = DistributedNode::new(test_config("127.0.0.1:10030"));
|
||||
let mut b = DistributedNode::new(test_config("127.0.0.1:10031"));
|
||||
let mut c = DistributedNode::new(test_config("127.0.0.1:10032"));
|
||||
|
||||
let a_id = a.node_id();
|
||||
let a_addr = a.listen_addr();
|
||||
let b_id = b.node_id();
|
||||
let b_addr = b.listen_addr();
|
||||
let c_id = c.node_id();
|
||||
let c_addr = c.listen_addr();
|
||||
|
||||
// B and C join A.
|
||||
let actions = b.join(&[a_addr]);
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a)];
|
||||
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
|
||||
let mut nodes = vec![(b_id, b_addr, &mut b)];
|
||||
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
|
||||
|
||||
let actions = c.join(&[a_addr]);
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a)];
|
||||
let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes);
|
||||
let mut nodes = vec![(c_id, c_addr, &mut c)];
|
||||
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
|
||||
|
||||
// B registers a name.
|
||||
let actor = ActorAddress::new_random();
|
||||
b.register_name("b-service".into(), actor);
|
||||
|
||||
// Propagate B's registration to A and C via mesh gossip.
|
||||
// B only knows A, so first B→A, then A→C carries it.
|
||||
for _ in 0..5 {
|
||||
// Each node ticks and delivers to all others.
|
||||
let actions = b.tick();
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a), (c_id, c_addr, &mut c)];
|
||||
let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes);
|
||||
let mut nodes = vec![(b_id, b_addr, &mut b)];
|
||||
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
|
||||
|
||||
let actions = a.tick();
|
||||
let mut nodes = vec![(b_id, b_addr, &mut b), (c_id, c_addr, &mut c)];
|
||||
let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes);
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a)];
|
||||
let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes);
|
||||
|
||||
let actions = c.tick();
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a), (b_id, b_addr, &mut b)];
|
||||
let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes);
|
||||
let mut nodes = vec![(c_id, c_addr, &mut c)];
|
||||
let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes);
|
||||
}
|
||||
|
||||
assert_eq!(a.resolve_name("b-service"), Some((actor, b_id)));
|
||||
assert_eq!(c.resolve_name("b-service"), Some((actor, b_id)));
|
||||
|
||||
// B dies — SWIM detects via timeout. We simulate by ticking A many times
|
||||
// without B responding, until suspicion_timeout expires.
|
||||
for _ in 0..20 {
|
||||
let actions = a.tick();
|
||||
// Don't deliver to B — it's "dead". Only deliver to C.
|
||||
let mut nodes = vec![(c_id, c_addr, &mut c)];
|
||||
let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes);
|
||||
let mut nodes = vec![(a_id, a_addr, &mut a)];
|
||||
let _ = deliver_actions(&responses, c_id, c_addr, &mut nodes);
|
||||
}
|
||||
|
||||
// After enough ticks, A should declare B dead, which tombstones "b-service".
|
||||
// Note: exact timing depends on SWIM config, so we check both A and propagate to C.
|
||||
let a_resolved = a.resolve_name("b-service");
|
||||
|
||||
if a_resolved.is_none() {
|
||||
// A has tombstoned it — propagate to C.
|
||||
gossip_rounds(&mut a, a_id, a_addr, &mut c, c_id, c_addr, 5);
|
||||
assert_eq!(c.resolve_name("b-service"), None, "C should see tombstone after B's death propagates");
|
||||
}
|
||||
// If SWIM hasn't declared death yet, the test still passes — the mechanism
|
||||
// is wired, just needs more ticks. The important thing: no panics, clean flow.
|
||||
}
|
||||
|
||||
// ─── Test 10: registry events emitted on change ─────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn registry_events_emitted_on_change() {
|
||||
let mut node = DistributedNode::new(test_config("127.0.0.1:10040"));
|
||||
let actor = ActorAddress::new_random();
|
||||
let node_id = node.node_id();
|
||||
|
||||
node.register_name("evt-test".into(), actor);
|
||||
node.unregister_name("evt-test");
|
||||
|
||||
let events = node.registry_events();
|
||||
assert_eq!(events.len(), 2);
|
||||
assert_eq!(
|
||||
events[0],
|
||||
RegistryEvent::Registered {
|
||||
name: "evt-test".into(),
|
||||
actor_addr: actor,
|
||||
node_id,
|
||||
}
|
||||
);
|
||||
assert!(matches!(
|
||||
&events[1],
|
||||
RegistryEvent::Unregistered { name, previous_addr }
|
||||
if name == "evt-test" && *previous_addr == actor
|
||||
));
|
||||
}
|
||||
|
||||
// ─── Test 11: tombstone GC removes old tombstones ──────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tombstone_gc_removes_old_tombstones() {
|
||||
let mut reg = ClusterRegistry::new(RegistryConfig {
|
||||
tombstone_ttl: 10,
|
||||
gc_interval: 1,
|
||||
..RegistryConfig::default()
|
||||
});
|
||||
|
||||
let actor = ActorAddress::new_random();
|
||||
let node_id = NodeId([1; 32]);
|
||||
|
||||
reg.register("gc-me".into(), actor, node_id, 1);
|
||||
reg.unregister("gc-me", node_id, 1);
|
||||
|
||||
// Tombstone exists.
|
||||
assert_eq!(reg.resolve("gc-me"), None);
|
||||
assert_eq!(reg.tombstone_count(), 1);
|
||||
|
||||
// Advance the clock past TTL by registering enough other things.
|
||||
// Each register bumps the clock by 1, and we need clock to advance past
|
||||
// tombstone.timestamp + tombstone_ttl.
|
||||
for i in 0..15 {
|
||||
let a = ActorAddress::new_random();
|
||||
reg.register(format!("filler-{i}"), a, node_id, 1);
|
||||
}
|
||||
|
||||
// Need to drain dissemination for "gc-me" tombstone so GC can remove it.
|
||||
for _ in 0..20 {
|
||||
reg.take_pending(100);
|
||||
}
|
||||
|
||||
// Now run GC.
|
||||
reg.gc_tick();
|
||||
|
||||
// The tombstone should be gone.
|
||||
assert_eq!(reg.tombstone_count(), 0, "tombstone should be GC'd after TTL");
|
||||
}
|
||||
|
||||
// ─── Test 12: gossip convergence with five nodes ────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn gossip_convergence_five_nodes() {
|
||||
let base_port = 10050;
|
||||
let mut nodes: Vec<DistributedNode> = (0..5)
|
||||
.map(|i| {
|
||||
DistributedNode::new(test_config(&format!("127.0.0.1:{}", base_port + i)))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Collect ids/addrs before joining (borrow gymnastics).
|
||||
let ids: Vec<NodeId> = nodes.iter().map(|n| n.node_id()).collect();
|
||||
let addrs: Vec<SocketAddr> = nodes.iter().map(|n| n.listen_addr()).collect();
|
||||
|
||||
// All join through node 0.
|
||||
for i in 1..5 {
|
||||
let actions = nodes[i].join(&[addrs[0]]);
|
||||
// Deliver join request to node 0.
|
||||
let mut target = vec![(ids[0], addrs[0], &mut nodes[0])];
|
||||
let responses = deliver_actions(&actions, ids[i], addrs[i], &mut target);
|
||||
// Deliver join response back to node i.
|
||||
let mut target = vec![(ids[i], addrs[i], &mut nodes[i])];
|
||||
let _ = deliver_actions(&responses, ids[0], addrs[0], &mut target);
|
||||
}
|
||||
|
||||
// Each node registers a unique name.
|
||||
let actors: Vec<ActorAddress> = (0..5).map(|_| ActorAddress::new_random()).collect();
|
||||
for i in 0..5 {
|
||||
nodes[i].register_name(format!("service-{i}"), actors[i]);
|
||||
}
|
||||
|
||||
// Run many gossip rounds between all pairs.
|
||||
for _round in 0..15 {
|
||||
for i in 0..5 {
|
||||
let tick_actions = nodes[i].tick();
|
||||
// Deliver to all other nodes.
|
||||
for j in 0..5 {
|
||||
if i == j { continue; }
|
||||
let mut target = vec![(ids[j], addrs[j], &mut nodes[j])];
|
||||
let responses = deliver_actions(&tick_actions, ids[i], addrs[i], &mut target);
|
||||
let mut target = vec![(ids[i], addrs[i], &mut nodes[i])];
|
||||
let _ = deliver_actions(&responses, ids[j], addrs[j], &mut target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// All 5 names should be resolvable on all 5 nodes.
|
||||
for i in 0..5 {
|
||||
for j in 0..5 {
|
||||
let result = nodes[i].resolve_name(&format!("service-{j}"));
|
||||
assert_eq!(
|
||||
result,
|
||||
Some((actors[j], ids[j])),
|
||||
"node {i} should resolve service-{j}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ crossbeam-queue = "0.3.12"
|
|||
ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] }
|
||||
crossterm = { version = "0.28", optional = true }
|
||||
distribution = { path = "../distribution", optional = true }
|
||||
swactor-command = { path = "../command" }
|
||||
|
||||
[dependencies.ctrlc]
|
||||
version = "3"
|
||||
|
|
|
|||
|
|
@ -291,6 +291,7 @@ fn main() {
|
|||
swim: swim_config.clone(),
|
||||
cache_capacity: if i == 0 { 1000 } else { 100 },
|
||||
republish_interval: 500,
|
||||
..Default::default()
|
||||
};
|
||||
let node = DistributedNode::new(config);
|
||||
node_ids.push(node.node_id());
|
||||
|
|
@ -448,6 +449,7 @@ fn main() {
|
|||
swim: swim_config.clone(),
|
||||
cache_capacity: 100,
|
||||
republish_interval: 500,
|
||||
..Default::default()
|
||||
};
|
||||
let revived = DistributedNode::new(config);
|
||||
let join_actions = revived.join(&[seed_addr]);
|
||||
|
|
@ -512,6 +514,7 @@ fn main() {
|
|||
swim: swim_config.clone(),
|
||||
cache_capacity: 100,
|
||||
republish_interval: 500,
|
||||
..Default::default()
|
||||
};
|
||||
let revived = DistributedNode::new(config);
|
||||
let join_actions = revived.join(&[seed_addr]);
|
||||
|
|
|
|||
|
|
@ -42,6 +42,12 @@ impl StatsCollector {
|
|||
}
|
||||
}
|
||||
|
||||
impl swactor_command::StatsEnricher for StatsCollector {
|
||||
fn enrich(&self, stats: &mut swactor::stats::RuntimeStats) {
|
||||
stats.actor_details = self.actor_details();
|
||||
}
|
||||
}
|
||||
|
||||
impl StatsHook for StatsCollector {
|
||||
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) {
|
||||
if let Some(slot) = self.slots.get(worker_id) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
//! Line-oriented diagnostic protocol for LLM-driven runtime investigation.
|
||||
//!
|
||||
//! Delegates all command logic to the `swactor-command` crate.
|
||||
//!
|
||||
//! Send text commands on stdin, receive JSON responses on stdout (one per line).
|
||||
//! All human-readable diagnostics go to stderr.
|
||||
//!
|
||||
|
|
@ -19,16 +21,17 @@
|
|||
use std::collections::HashMap;
|
||||
use std::io::{self, BufRead, Write};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::Serialize;
|
||||
use swactor::runtime::Runtime;
|
||||
use swactor::stats::RuntimeStats;
|
||||
use swactor_command::{CommandContext, CommandRouter};
|
||||
|
||||
use crate::collector::StatsCollector;
|
||||
|
||||
/// Run the investigate REPL. Blocks until stdin is closed or `quit` is received.
|
||||
pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) -> io::Result<()> {
|
||||
let router = CommandRouter::with_builtins();
|
||||
let ctx = CommandContext::with_enricher(runtime, collector);
|
||||
|
||||
let stdin = io::stdin();
|
||||
let mut stdout = io::stdout();
|
||||
|
||||
|
|
@ -40,18 +43,14 @@ pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) ->
|
|||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
let cmd = parts[0];
|
||||
let args = &parts[1..];
|
||||
|
||||
if cmd == "quit" || cmd == "exit" {
|
||||
if line == "quit" || line == "exit" {
|
||||
break;
|
||||
}
|
||||
|
||||
let response = dispatch_repl(cmd, args, &runtime, &collector);
|
||||
let req = swactor_command::parse_line(line);
|
||||
let resp = router.dispatch(&req, &ctx);
|
||||
|
||||
stdout.write_all(response.as_bytes())?;
|
||||
stdout.write_all(resp.to_json_line().as_bytes())?;
|
||||
stdout.write_all(b"\n")?;
|
||||
stdout.flush()?;
|
||||
}
|
||||
|
|
@ -59,501 +58,14 @@ pub fn run_investigate(runtime: Arc<Runtime>, collector: Arc<StatsCollector>) ->
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn dispatch_repl(cmd: &str, args: &[&str], runtime: &Runtime, collector: &StatsCollector) -> String {
|
||||
match cmd {
|
||||
"help" => cmd_help(),
|
||||
"overview" => cmd_overview(runtime, collector),
|
||||
"workers" => cmd_workers(runtime),
|
||||
"worker" => cmd_worker(runtime, collector, args),
|
||||
"actors" => cmd_actors(runtime, collector, args),
|
||||
"actor" => cmd_actor(runtime, collector, args),
|
||||
"hot" => cmd_hot(runtime, collector, args),
|
||||
"phases" => cmd_phases(runtime, args),
|
||||
"diff" => cmd_diff(runtime, collector, args),
|
||||
_ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatch an investigate command from HTTP query parameters.
|
||||
///
|
||||
/// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command function.
|
||||
/// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command.
|
||||
pub fn dispatch_command(
|
||||
cmd: &str,
|
||||
params: &HashMap<String, String>,
|
||||
runtime: &Runtime,
|
||||
collector: &StatsCollector,
|
||||
router: &CommandRouter,
|
||||
ctx: &CommandContext,
|
||||
) -> String {
|
||||
match cmd {
|
||||
"help" => cmd_help(),
|
||||
"overview" => cmd_overview(runtime, collector),
|
||||
"workers" => cmd_workers(runtime),
|
||||
"worker" => {
|
||||
let id = params.get("id").map(|s| s.as_str()).unwrap_or("");
|
||||
cmd_worker(runtime, collector, &[id])
|
||||
}
|
||||
"actors" => {
|
||||
let mut args = Vec::new();
|
||||
if let Some(sort) = params.get("sort") {
|
||||
args.push("--sort");
|
||||
args.push(sort.as_str());
|
||||
}
|
||||
if let Some(limit) = params.get("limit") {
|
||||
args.push("--limit");
|
||||
args.push(limit.as_str());
|
||||
}
|
||||
if let Some(worker) = params.get("worker") {
|
||||
args.push("--worker");
|
||||
args.push(worker.as_str());
|
||||
}
|
||||
cmd_actors(runtime, collector, &args)
|
||||
}
|
||||
"actor" => {
|
||||
let prefix = params.get("prefix").map(|s| s.as_str()).unwrap_or("");
|
||||
cmd_actor(runtime, collector, &[prefix])
|
||||
}
|
||||
"hot" => {
|
||||
let n = params.get("n").map(|s| s.as_str()).unwrap_or("10");
|
||||
cmd_hot(runtime, collector, &[n])
|
||||
}
|
||||
"phases" => {
|
||||
match params.get("worker") {
|
||||
Some(w) => cmd_phases(runtime, &[w.as_str()]),
|
||||
None => cmd_phases(runtime, &[]),
|
||||
}
|
||||
}
|
||||
"diff" => {
|
||||
let secs = params.get("seconds").map(|s| s.as_str()).unwrap_or("");
|
||||
cmd_diff(runtime, collector, &[secs])
|
||||
}
|
||||
_ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
fn ok_response(cmd: &str, data: impl Serialize) -> String {
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"ok": true,
|
||||
"command": cmd,
|
||||
"data": data,
|
||||
}))
|
||||
.unwrap_or_else(|e| err_response(cmd, &format!("serialization error: {e}")))
|
||||
}
|
||||
|
||||
fn err_response(cmd: &str, msg: &str) -> String {
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"ok": false,
|
||||
"command": cmd,
|
||||
"error": msg,
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn format_addr(addr: &swactor::actor::ActorAddress) -> String {
|
||||
format!("{addr}")
|
||||
}
|
||||
|
||||
fn full_hex(addr: &swactor::actor::ActorAddress) -> String {
|
||||
addr.0.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
fn enriched_stats(rt: &Runtime, col: &StatsCollector) -> RuntimeStats {
|
||||
let mut s = rt.stats();
|
||||
col.enrich(&mut s);
|
||||
s
|
||||
}
|
||||
|
||||
// ── Commands ────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn cmd_help() -> String {
|
||||
ok_response(
|
||||
"help",
|
||||
serde_json::json!({
|
||||
"commands": [
|
||||
{"name": "overview", "usage": "overview", "description": "Summary: worker count, actor count, total messages, mailbox depth, panics"},
|
||||
{"name": "workers", "usage": "workers", "description": "Per-worker stats: actors, mailbox depth, messages, sends (local/cross/inbox), panics"},
|
||||
{"name": "worker", "usage": "worker <id>", "description": "Single worker detail with tick-phase timing breakdown"},
|
||||
{"name": "actors", "usage": "actors [--sort mailbox|worker|address] [--limit N] [--worker W]", "description": "List actors with optional sorting, limit, and worker filter"},
|
||||
{"name": "actor", "usage": "actor <hex_prefix>", "description": "Find actor(s) whose address starts with the given hex prefix"},
|
||||
{"name": "hot", "usage": "hot [N]", "description": "Top N actors by mailbox depth (default 10)"},
|
||||
{"name": "phases", "usage": "phases [worker_id]", "description": "Tick-phase time breakdown (all workers or one)"},
|
||||
{"name": "diff", "usage": "diff <seconds>", "description": "Collect two snapshots N seconds apart, report deltas and rates"},
|
||||
{"name": "quit", "usage": "quit", "description": "Exit the investigate session"},
|
||||
]
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn cmd_overview(rt: &Runtime, col: &StatsCollector) -> String {
|
||||
let stats = enriched_stats(rt, col);
|
||||
let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum();
|
||||
let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum();
|
||||
let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum();
|
||||
let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum();
|
||||
let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum();
|
||||
let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum();
|
||||
|
||||
ok_response(
|
||||
"overview",
|
||||
serde_json::json!({
|
||||
"workers": stats.num_workers,
|
||||
"actors": stats.actor_details.len(),
|
||||
"total_messages_processed": total_msgs,
|
||||
"total_mailbox_depth": total_mailbox,
|
||||
"total_panics": total_panics,
|
||||
"total_type_mismatches": total_type_mismatches,
|
||||
"sends": {
|
||||
"local": total_local,
|
||||
"cross_worker": total_cross,
|
||||
"inbox": total_inbox,
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn cmd_workers(rt: &Runtime) -> String {
|
||||
let stats = rt.stats();
|
||||
let workers: Vec<_> = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|w| {
|
||||
serde_json::json!({
|
||||
"id": w.id,
|
||||
"actors": w.num_actors,
|
||||
"mailbox_depth": w.mailbox_depth,
|
||||
"messages_processed": w.messages_processed,
|
||||
"local_sends": w.local_sends,
|
||||
"cross_sends": w.cross_sends,
|
||||
"inbox_sends": w.inbox_sends,
|
||||
"type_mismatches": w.type_mismatches,
|
||||
"panics": w.panics,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
ok_response("workers", workers)
|
||||
}
|
||||
|
||||
pub fn cmd_worker(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
|
||||
let id: usize = match args.first().and_then(|s| s.parse().ok()) {
|
||||
Some(id) => id,
|
||||
None => return err_response("worker", "usage: worker <id>"),
|
||||
};
|
||||
|
||||
let stats = enriched_stats(rt, col);
|
||||
let w = match stats.workers.iter().find(|w| w.id == id) {
|
||||
Some(w) => w,
|
||||
None => {
|
||||
return err_response(
|
||||
"worker",
|
||||
&format!("worker {id} not found (have 0..{})", stats.num_workers),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
// Tick phase breakdown for this worker
|
||||
let timings = stats.tick_timings.get(id).cloned().unwrap_or_default();
|
||||
let phase_breakdown = compute_phase_breakdown(&timings);
|
||||
|
||||
let actors_on_worker: Vec<_> = stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.filter(|a| a.worker_id == id)
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
ok_response(
|
||||
"worker",
|
||||
serde_json::json!({
|
||||
"id": w.id,
|
||||
"actors": w.num_actors,
|
||||
"mailbox_depth": w.mailbox_depth,
|
||||
"messages_processed": w.messages_processed,
|
||||
"local_sends": w.local_sends,
|
||||
"cross_sends": w.cross_sends,
|
||||
"inbox_sends": w.inbox_sends,
|
||||
"type_mismatches": w.type_mismatches,
|
||||
"panics": w.panics,
|
||||
"tick_phases": phase_breakdown,
|
||||
"actor_details": actors_on_worker,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn cmd_actors(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
|
||||
let stats = enriched_stats(rt, col);
|
||||
let mut actors = stats.actor_details.clone();
|
||||
|
||||
// Parse flags
|
||||
let mut sort_by = "mailbox";
|
||||
let mut limit: usize = usize::MAX;
|
||||
let mut worker_filter: Option<usize> = None;
|
||||
let mut i = 0;
|
||||
while i < args.len() {
|
||||
match args[i] {
|
||||
"--sort" if i + 1 < args.len() => {
|
||||
sort_by = args[i + 1];
|
||||
i += 2;
|
||||
}
|
||||
"--limit" if i + 1 < args.len() => {
|
||||
limit = args[i + 1].parse().unwrap_or(usize::MAX);
|
||||
i += 2;
|
||||
}
|
||||
"--worker" if i + 1 < args.len() => {
|
||||
worker_filter = args[i + 1].parse().ok();
|
||||
i += 2;
|
||||
}
|
||||
_ => {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(wid) = worker_filter {
|
||||
actors.retain(|a| a.worker_id == wid);
|
||||
}
|
||||
|
||||
match sort_by {
|
||||
"mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)),
|
||||
"worker" => actors.sort_by_key(|a| a.worker_id),
|
||||
"address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)),
|
||||
other => return err_response("actors", &format!("unknown sort field `{other}` — use mailbox|worker|address")),
|
||||
}
|
||||
|
||||
actors.truncate(limit);
|
||||
|
||||
let rows: Vec<_> = actors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"address_full": full_hex(&a.address),
|
||||
"worker_id": a.worker_id,
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
ok_response(
|
||||
"actors",
|
||||
serde_json::json!({
|
||||
"total": stats.actor_details.len(),
|
||||
"returned": rows.len(),
|
||||
"sort": sort_by,
|
||||
"actors": rows,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn cmd_actor(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
|
||||
let prefix = match args.first() {
|
||||
Some(p) => *p,
|
||||
None => return err_response("actor", "usage: actor <hex_prefix>"),
|
||||
};
|
||||
|
||||
let stats = enriched_stats(rt, col);
|
||||
let matches: Vec<_> = stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.filter(|a| full_hex(&a.address).starts_with(prefix))
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"address_full": full_hex(&a.address),
|
||||
"worker_id": a.worker_id,
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
ok_response(
|
||||
"actor",
|
||||
serde_json::json!({
|
||||
"prefix": prefix,
|
||||
"matches": matches.len(),
|
||||
"actors": matches,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn cmd_hot(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
|
||||
let n: usize = args.first().and_then(|s| s.parse().ok()).unwrap_or(10);
|
||||
let stats = enriched_stats(rt, col);
|
||||
|
||||
let mut actors = stats.actor_details.clone();
|
||||
actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth));
|
||||
actors.truncate(n);
|
||||
|
||||
let rows: Vec<_> = actors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
serde_json::json!({
|
||||
"address": format_addr(&a.address),
|
||||
"address_full": full_hex(&a.address),
|
||||
"worker_id": a.worker_id,
|
||||
"mailbox_depth": a.mailbox_depth,
|
||||
"last_msg_type": a.last_msg_type,
|
||||
"messages_processed": a.messages_processed,
|
||||
"poisoned": a.poisoned,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
ok_response("hot", rows)
|
||||
}
|
||||
|
||||
pub fn cmd_phases(rt: &Runtime, args: &[&str]) -> String {
|
||||
let stats = rt.stats();
|
||||
|
||||
let worker_filter: Option<usize> = args.first().and_then(|s| s.parse().ok());
|
||||
|
||||
let phase_names = [
|
||||
"spawn_drain",
|
||||
"transfer_drain",
|
||||
"tick_all",
|
||||
"spawn_drain_2",
|
||||
"pending_local",
|
||||
"stats_publish",
|
||||
];
|
||||
|
||||
let mut results = Vec::new();
|
||||
for (i, timings) in stats.tick_timings.iter().enumerate() {
|
||||
if let Some(wid) = worker_filter {
|
||||
if i != wid {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let breakdown = compute_phase_breakdown(timings);
|
||||
results.push(serde_json::json!({
|
||||
"worker_id": i,
|
||||
"ticks_sampled": timings.len(),
|
||||
"phases": breakdown,
|
||||
"phase_names": phase_names,
|
||||
}));
|
||||
}
|
||||
|
||||
ok_response("phases", results)
|
||||
}
|
||||
|
||||
pub fn cmd_diff(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String {
|
||||
let secs: f64 = match args.first().and_then(|s| s.parse().ok()) {
|
||||
Some(s) if s > 0.0 && s <= 30.0 => s,
|
||||
Some(_) => return err_response("diff", "seconds must be between 0 and 30"),
|
||||
None => return err_response("diff", "usage: diff <seconds>"),
|
||||
};
|
||||
|
||||
let before = enriched_stats(rt, col);
|
||||
let t0 = Instant::now();
|
||||
std::thread::sleep(Duration::from_secs_f64(secs));
|
||||
let after = enriched_stats(rt, col);
|
||||
let elapsed = t0.elapsed().as_secs_f64();
|
||||
|
||||
let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum();
|
||||
let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum();
|
||||
let delta_msgs = msgs_after.saturating_sub(msgs_before);
|
||||
|
||||
let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum();
|
||||
let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum();
|
||||
let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum();
|
||||
let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum();
|
||||
|
||||
let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum();
|
||||
|
||||
let per_worker: Vec<_> = after
|
||||
.workers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, w)| {
|
||||
let prev = before.workers.get(i);
|
||||
let d = prev
|
||||
.map(|p| w.messages_processed.saturating_sub(p.messages_processed))
|
||||
.unwrap_or(0);
|
||||
serde_json::json!({
|
||||
"worker_id": i,
|
||||
"delta_messages": d,
|
||||
"msg_per_sec": d as f64 / elapsed,
|
||||
"actors_before": prev.map(|p| p.num_actors).unwrap_or(0),
|
||||
"actors_after": w.num_actors,
|
||||
"mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0),
|
||||
"mailbox_after": w.mailbox_depth,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
ok_response(
|
||||
"diff",
|
||||
serde_json::json!({
|
||||
"elapsed_s": elapsed,
|
||||
"actors_before": before.actor_details.len(),
|
||||
"actors_after": after.actor_details.len(),
|
||||
"delta_messages": delta_msgs,
|
||||
"msg_per_sec": delta_msgs as f64 / elapsed,
|
||||
"delta_local_sends": local_after.saturating_sub(local_before),
|
||||
"delta_cross_sends": cross_after.saturating_sub(cross_before),
|
||||
"mailbox_before": mailbox_before,
|
||||
"mailbox_after": mailbox_after,
|
||||
"per_worker": per_worker,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// ── Phase breakdown helper ──────────────────────────────────────────────
|
||||
|
||||
fn compute_phase_breakdown(
|
||||
timings: &[swactor::stats::TickTiming],
|
||||
) -> serde_json::Value {
|
||||
if timings.is_empty() {
|
||||
return serde_json::json!({
|
||||
"ticks": 0,
|
||||
"active_pct": 0.0,
|
||||
"avg_tick_us": 0.0,
|
||||
"phases_us": [0, 0, 0, 0, 0, 0],
|
||||
"phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
||||
});
|
||||
}
|
||||
|
||||
let n = timings.len();
|
||||
let active = timings.iter().filter(|t| t.did_work).count();
|
||||
let active_pct = (active as f64 / n as f64) * 100.0;
|
||||
|
||||
let mut phase_sums = [0u64; 6];
|
||||
for t in timings {
|
||||
for (i, &us) in t.phase_us.iter().enumerate() {
|
||||
phase_sums[i] += us;
|
||||
}
|
||||
}
|
||||
let total_us: u64 = phase_sums.iter().sum();
|
||||
let avg_tick_us = total_us as f64 / n as f64;
|
||||
|
||||
let phases_pct: Vec<f64> = if total_us == 0 {
|
||||
vec![0.0; 6]
|
||||
} else {
|
||||
phase_sums
|
||||
.iter()
|
||||
.map(|&s| (s as f64 / total_us as f64) * 100.0)
|
||||
.collect()
|
||||
};
|
||||
|
||||
serde_json::json!({
|
||||
"ticks": n,
|
||||
"active_pct": active_pct,
|
||||
"avg_tick_us": avg_tick_us,
|
||||
"phases_us": phase_sums,
|
||||
"phases_pct": phases_pct,
|
||||
})
|
||||
let req = swactor_command::from_query_params(params);
|
||||
router.dispatch(&req, ctx).to_json_line()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ use swactor::runtime::Runtime;
|
|||
use crate::actors_html::ACTORS_HTML;
|
||||
use crate::collector::StatsCollector;
|
||||
use crate::dashboard_html::DASHBOARD_HTML;
|
||||
use crate::investigate;
|
||||
use crate::layer::EventStore;
|
||||
use crate::trace::RuntimeTrace;
|
||||
|
||||
|
|
@ -127,6 +126,7 @@ pub(crate) fn spawn_http_server(
|
|||
let addr = format!("0.0.0.0:{port}");
|
||||
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
|
||||
let server = Arc::new(server);
|
||||
let cmd_router = Arc::new(swactor_command::CommandRouter::with_builtins());
|
||||
|
||||
for _ in 0..4 {
|
||||
let server = Arc::clone(&server);
|
||||
|
|
@ -134,6 +134,7 @@ pub(crate) fn spawn_http_server(
|
|||
let runtime = Arc::clone(&runtime);
|
||||
let collector = Arc::clone(&collector);
|
||||
let shutdown = Arc::clone(&shutdown);
|
||||
let cmd_router = Arc::clone(&cmd_router);
|
||||
#[cfg(feature = "distribution")]
|
||||
let distribution = Arc::clone(&distribution);
|
||||
thread::spawn(move || {
|
||||
|
|
@ -174,6 +175,7 @@ pub(crate) fn spawn_http_server(
|
|||
&url,
|
||||
Arc::clone(&runtime),
|
||||
Arc::clone(&collector),
|
||||
Arc::clone(&cmd_router),
|
||||
);
|
||||
}
|
||||
_ => respond_404(request),
|
||||
|
|
@ -283,21 +285,33 @@ fn handle_investigate_api(
|
|||
url: &str,
|
||||
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||
collector: Arc<Mutex<Option<Arc<StatsCollector>>>>,
|
||||
cmd_router: Arc<swactor_command::CommandRouter>,
|
||||
) {
|
||||
let params = parse_query_string(url);
|
||||
let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help");
|
||||
|
||||
let maybe_rt = runtime.lock().unwrap().clone();
|
||||
let maybe_col = collector.lock().unwrap().clone();
|
||||
|
||||
let json = match (maybe_rt, maybe_col) {
|
||||
(Some(rt), Some(col)) => investigate::dispatch_command(cmd, ¶ms, &rt, &col),
|
||||
_ => serde_json::json!({
|
||||
"ok": false,
|
||||
"command": cmd,
|
||||
"error": "runtime not attached yet"
|
||||
})
|
||||
.to_string(),
|
||||
(Some(rt), Some(col)) => {
|
||||
let ctx = swactor_command::CommandContext::with_enricher(rt, col);
|
||||
let req = swactor_command::from_query_params(¶ms);
|
||||
cmd_router.dispatch(&req, &ctx).to_json_line()
|
||||
}
|
||||
(Some(rt), None) => {
|
||||
let ctx = swactor_command::CommandContext::new(rt);
|
||||
let req = swactor_command::from_query_params(¶ms);
|
||||
cmd_router.dispatch(&req, &ctx).to_json_line()
|
||||
}
|
||||
_ => {
|
||||
let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help");
|
||||
serde_json::json!({
|
||||
"ok": false,
|
||||
"command": cmd,
|
||||
"error": "runtime not attached yet"
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
};
|
||||
|
||||
let response = tiny_http::Response::from_string(json).with_header(
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
|||
swim: config.swim.clone(),
|
||||
cache_capacity: config.cache_capacity,
|
||||
republish_interval: 50,
|
||||
..Default::default()
|
||||
};
|
||||
let node = DistributedNode::new(node_config);
|
||||
node_ids.push(node.node_id());
|
||||
|
|
@ -184,6 +185,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace {
|
|||
swim: config.swim.clone(),
|
||||
cache_capacity: config.cache_capacity,
|
||||
republish_interval: 50,
|
||||
..Default::default()
|
||||
};
|
||||
let revived = DistributedNode::new(node_config);
|
||||
// Rejoin the cluster.
|
||||
|
|
|
|||
12
crates/wasm-actor/Cargo.toml
Normal file
12
crates/wasm-actor/Cargo.toml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
[package]
|
||||
name = "swactor-wasm-actor"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
swactor = { path = "../.." }
|
||||
wasmtime = "29"
|
||||
|
||||
[dev-dependencies]
|
||||
swactor = { path = "../..", features = ["getrandom"] }
|
||||
wat = "1"
|
||||
59
crates/wasm-actor/src/actor.rs
Normal file
59
crates/wasm-actor/src/actor.rs
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
use swactor::actor::{ActorInterface, Ctx};
|
||||
use wasmtime::{Memory, Store, TypedFunc};
|
||||
|
||||
use crate::ByteMessage;
|
||||
|
||||
/// State accessible to host functions during guest execution.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct HostState {
|
||||
pub outbox: Vec<(swactor::actor::ActorAddress, Vec<u8>)>,
|
||||
}
|
||||
|
||||
/// An actor whose logic is defined by a WebAssembly guest module.
|
||||
///
|
||||
/// Messages arrive as [`ByteMessage`], are copied into Wasm linear memory,
|
||||
/// and processed by the guest's `handle` export. The guest can send messages
|
||||
/// back via the `swactor.send` host import.
|
||||
pub struct WasmActor {
|
||||
pub(crate) store: Store<HostState>,
|
||||
pub(crate) memory: Memory,
|
||||
pub(crate) alloc: TypedFunc<i32, i32>,
|
||||
pub(crate) handle: TypedFunc<(i32, i32), ()>,
|
||||
}
|
||||
|
||||
impl ActorInterface for WasmActor {
|
||||
type Incoming = ByteMessage;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, msg: ByteMessage) {
|
||||
let bytes = &msg.0;
|
||||
let len: i32 = match i32::try_from(bytes.len()) {
|
||||
Ok(n) => n,
|
||||
Err(_) => return, // message too large for i32 ABI
|
||||
};
|
||||
|
||||
// 1. Allocate space in guest memory
|
||||
let ptr = match self.alloc.call(&mut self.store, len) {
|
||||
Ok(ptr) if ptr < 0 => return, // invalid pointer
|
||||
Ok(0) if len > 0 => return, // OOM — drop message
|
||||
Ok(ptr) => ptr,
|
||||
Err(_) => return, // alloc trapped — drop message
|
||||
};
|
||||
|
||||
// 2. Write message bytes into guest memory
|
||||
self.memory.data_mut(&mut self.store)
|
||||
[ptr as usize..(ptr as usize + bytes.len())]
|
||||
.copy_from_slice(bytes);
|
||||
|
||||
// 3. Call guest handle
|
||||
if self.handle.call(&mut self.store, (ptr, len)).is_err() {
|
||||
return; // handle trapped — drop message, keep actor alive
|
||||
}
|
||||
|
||||
// 4. Drain outbox → send via ctx
|
||||
let outbox: Vec<_> = self.store.data_mut().outbox.drain(..).collect();
|
||||
for (dest, payload) in outbox {
|
||||
let _ = ctx.send(dest, ByteMessage(payload));
|
||||
}
|
||||
}
|
||||
}
|
||||
109
crates/wasm-actor/src/builder.rs
Normal file
109
crates/wasm-actor/src/builder.rs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
use swactor::actor::ActorAddress;
|
||||
use wasmtime::{Linker, Module, Store, TypedFunc};
|
||||
|
||||
use crate::actor::{HostState, WasmActor};
|
||||
use crate::engine::SharedEngine;
|
||||
use crate::error::WasmActorError;
|
||||
|
||||
/// Compiles a Wasm module and produces a ready-to-use [`WasmActor`].
|
||||
pub struct WasmActorBuilder {
|
||||
engine: SharedEngine,
|
||||
wasm_bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl WasmActorBuilder {
|
||||
pub fn new(engine: SharedEngine, wasm_bytes: impl Into<Vec<u8>>) -> Self {
|
||||
Self {
|
||||
engine,
|
||||
wasm_bytes: wasm_bytes.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile the module, link host functions, and instantiate.
|
||||
pub fn build(self) -> Result<WasmActor, WasmActorError> {
|
||||
let engine = self.engine.inner();
|
||||
let module = Module::new(engine, &self.wasm_bytes)?;
|
||||
|
||||
let mut linker: Linker<HostState> = Linker::new(engine);
|
||||
Self::link_send(&mut linker)?;
|
||||
|
||||
let mut store = Store::new(engine, HostState::default());
|
||||
let instance = linker.instantiate(&mut store, &module)?;
|
||||
|
||||
// Extract required exports
|
||||
let memory = instance
|
||||
.get_memory(&mut store, "memory")
|
||||
.ok_or(WasmActorError::MissingExport("memory"))?;
|
||||
|
||||
let alloc: TypedFunc<i32, i32> = instance
|
||||
.get_typed_func(&mut store, "alloc")
|
||||
.map_err(|_| WasmActorError::MissingExport("alloc"))?;
|
||||
|
||||
let handle: TypedFunc<(i32, i32), ()> = instance
|
||||
.get_typed_func(&mut store, "handle")
|
||||
.map_err(|_| WasmActorError::MissingExport("handle"))?;
|
||||
|
||||
Ok(WasmActor {
|
||||
store,
|
||||
memory,
|
||||
alloc,
|
||||
handle,
|
||||
})
|
||||
}
|
||||
|
||||
/// Link the `swactor.send` host import.
|
||||
fn link_send(linker: &mut Linker<HostState>) -> Result<(), WasmActorError> {
|
||||
linker.func_wrap(
|
||||
"swactor",
|
||||
"send",
|
||||
|mut caller: wasmtime::Caller<'_, HostState>,
|
||||
dest_ptr: i32,
|
||||
payload_ptr: i32,
|
||||
payload_len: i32|
|
||||
-> Result<(), wasmtime::Error> {
|
||||
let mem = caller
|
||||
.get_export("memory")
|
||||
.and_then(|e| e.into_memory())
|
||||
.ok_or_else(|| wasmtime::Error::msg("guest must export memory"))?;
|
||||
let data = mem.data(&caller);
|
||||
let mem_len = data.len();
|
||||
|
||||
// Validate non-negative arguments
|
||||
if dest_ptr < 0 || payload_ptr < 0 || payload_len < 0 {
|
||||
return Err(wasmtime::Error::msg(
|
||||
"negative argument in swactor.send",
|
||||
));
|
||||
}
|
||||
|
||||
let dest_ptr = dest_ptr as usize;
|
||||
let payload_ptr = payload_ptr as usize;
|
||||
let payload_len = payload_len as usize;
|
||||
|
||||
// Bounds-check with overflow protection
|
||||
let dest_end = dest_ptr
|
||||
.checked_add(32)
|
||||
.ok_or_else(|| wasmtime::Error::msg("dest_ptr overflow"))?;
|
||||
let payload_end = payload_ptr
|
||||
.checked_add(payload_len)
|
||||
.ok_or_else(|| wasmtime::Error::msg("payload range overflow"))?;
|
||||
if dest_end > mem_len || payload_end > mem_len {
|
||||
return Err(wasmtime::Error::msg(
|
||||
"out-of-bounds memory access in swactor.send",
|
||||
));
|
||||
}
|
||||
|
||||
// Read 32-byte destination address
|
||||
let mut addr_bytes = [0u8; 32];
|
||||
addr_bytes.copy_from_slice(&data[dest_ptr..dest_end]);
|
||||
let dest = ActorAddress(addr_bytes);
|
||||
|
||||
// Read payload
|
||||
let payload = data[payload_ptr..payload_end].to_vec();
|
||||
|
||||
caller.data_mut().outbox.push((dest, payload));
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
35
crates/wasm-actor/src/engine.rs
Normal file
35
crates/wasm-actor/src/engine.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use wasmtime::Engine;
|
||||
|
||||
/// A shared, cheaply-cloneable Wasm engine.
|
||||
///
|
||||
/// Created once and reused across multiple [`WasmActor`](crate::WasmActor) instances.
|
||||
/// Configured with maximum sandboxing — no threads, no SIMD, no reference types.
|
||||
#[derive(Clone)]
|
||||
pub struct SharedEngine(Arc<Engine>);
|
||||
|
||||
impl std::fmt::Debug for SharedEngine {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_tuple("SharedEngine").field(&"<Engine>").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl SharedEngine {
|
||||
/// Create a new engine with sandboxed defaults.
|
||||
pub fn new() -> Result<Self, wasmtime::Error> {
|
||||
let mut config = wasmtime::Config::new();
|
||||
config.wasm_threads(false);
|
||||
config.wasm_simd(false);
|
||||
config.wasm_relaxed_simd(false);
|
||||
config.wasm_reference_types(false);
|
||||
config.wasm_multi_value(false);
|
||||
config.wasm_bulk_memory(true);
|
||||
let engine = Engine::new(&config)?;
|
||||
Ok(Self(Arc::new(engine)))
|
||||
}
|
||||
|
||||
pub(crate) fn inner(&self) -> &Engine {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
27
crates/wasm-actor/src/error.rs
Normal file
27
crates/wasm-actor/src/error.rs
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
use std::fmt;
|
||||
|
||||
/// Errors that can occur when building or running a WasmActor.
|
||||
#[derive(Debug)]
|
||||
pub enum WasmActorError {
|
||||
/// A required export is missing from the Wasm module.
|
||||
MissingExport(&'static str),
|
||||
/// The Wasm module failed to compile or instantiate.
|
||||
Wasmtime(wasmtime::Error),
|
||||
}
|
||||
|
||||
impl fmt::Display for WasmActorError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::MissingExport(name) => write!(f, "missing required export: `{name}`"),
|
||||
Self::Wasmtime(e) => write!(f, "wasmtime error: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for WasmActorError {}
|
||||
|
||||
impl From<wasmtime::Error> for WasmActorError {
|
||||
fn from(e: wasmtime::Error) -> Self {
|
||||
Self::Wasmtime(e)
|
||||
}
|
||||
}
|
||||
13
crates/wasm-actor/src/lib.rs
Normal file
13
crates/wasm-actor/src/lib.rs
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
mod actor;
|
||||
mod builder;
|
||||
mod engine;
|
||||
mod error;
|
||||
|
||||
pub use actor::WasmActor;
|
||||
pub use builder::WasmActorBuilder;
|
||||
pub use engine::SharedEngine;
|
||||
pub use error::WasmActorError;
|
||||
|
||||
/// A message carrying raw bytes, suitable for passing to/from Wasm guests.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ByteMessage(pub Vec<u8>);
|
||||
7
crates/wasm-actor/tests/guests/double/Cargo.lock
generated
Normal file
7
crates/wasm-actor/tests/guests/double/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "double-guest"
|
||||
version = "0.1.0"
|
||||
13
crates/wasm-actor/tests/guests/double/Cargo.toml
Normal file
13
crates/wasm-actor/tests/guests/double/Cargo.toml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
[workspace]
|
||||
|
||||
[package]
|
||||
name = "double-guest"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[profile.release]
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
63
crates/wasm-actor/tests/guests/double/src/lib.rs
Normal file
63
crates/wasm-actor/tests/guests/double/src/lib.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
#![no_std]
|
||||
|
||||
use core::cell::UnsafeCell;
|
||||
use core::panic::PanicInfo;
|
||||
|
||||
// --- bump allocator ---
|
||||
const HEAP_SIZE: usize = 65536;
|
||||
|
||||
struct BumpAlloc {
|
||||
heap: UnsafeCell<[u8; HEAP_SIZE]>,
|
||||
offset: UnsafeCell<usize>,
|
||||
}
|
||||
|
||||
unsafe impl Sync for BumpAlloc {}
|
||||
|
||||
static ALLOC: BumpAlloc = BumpAlloc {
|
||||
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
|
||||
offset: UnsafeCell::new(0),
|
||||
};
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn alloc(size: i32) -> i32 {
|
||||
unsafe {
|
||||
let offset = &mut *ALLOC.offset.get();
|
||||
let heap = &mut *ALLOC.heap.get();
|
||||
let align = 8;
|
||||
let start = (*offset + align - 1) & !(align - 1);
|
||||
let end = start + size as usize;
|
||||
if end > heap.len() {
|
||||
return 0; // OOM
|
||||
}
|
||||
*offset = end;
|
||||
heap.as_ptr().add(start) as i32
|
||||
}
|
||||
}
|
||||
|
||||
// --- host import ---
|
||||
#[link(wasm_import_module = "swactor")]
|
||||
unsafe extern "C" {
|
||||
#[link_name = "send"]
|
||||
fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32);
|
||||
}
|
||||
|
||||
/// Message format: first 32 bytes = destination address, rest = payload.
|
||||
/// Sends the payload back twice to demonstrate multi-send.
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn handle(ptr: i32, len: i32) {
|
||||
if len < 32 {
|
||||
return;
|
||||
}
|
||||
let dest_ptr = ptr;
|
||||
let payload_ptr = ptr + 32;
|
||||
let payload_len = len - 32;
|
||||
unsafe {
|
||||
host_send(dest_ptr, payload_ptr, payload_len);
|
||||
host_send(dest_ptr, payload_ptr, payload_len);
|
||||
}
|
||||
}
|
||||
|
||||
#[panic_handler]
|
||||
fn panic(_info: &PanicInfo) -> ! {
|
||||
loop {}
|
||||
}
|
||||
7
crates/wasm-actor/tests/guests/echo/Cargo.lock
generated
Normal file
7
crates/wasm-actor/tests/guests/echo/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "echo-guest"
|
||||
version = "0.1.0"
|
||||
13
crates/wasm-actor/tests/guests/echo/Cargo.toml
Normal file
13
crates/wasm-actor/tests/guests/echo/Cargo.toml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
[workspace]
|
||||
|
||||
[package]
|
||||
name = "echo-guest"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[profile.release]
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
62
crates/wasm-actor/tests/guests/echo/src/lib.rs
Normal file
62
crates/wasm-actor/tests/guests/echo/src/lib.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
#![no_std]
|
||||
|
||||
use core::cell::UnsafeCell;
|
||||
use core::panic::PanicInfo;
|
||||
|
||||
// --- bump allocator ---
|
||||
const HEAP_SIZE: usize = 65536;
|
||||
|
||||
struct BumpAlloc {
|
||||
heap: UnsafeCell<[u8; HEAP_SIZE]>,
|
||||
offset: UnsafeCell<usize>,
|
||||
}
|
||||
|
||||
unsafe impl Sync for BumpAlloc {}
|
||||
|
||||
static ALLOC: BumpAlloc = BumpAlloc {
|
||||
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
|
||||
offset: UnsafeCell::new(0),
|
||||
};
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn alloc(size: i32) -> i32 {
|
||||
unsafe {
|
||||
let offset = &mut *ALLOC.offset.get();
|
||||
let heap = &mut *ALLOC.heap.get();
|
||||
let align = 8;
|
||||
let start = (*offset + align - 1) & !(align - 1);
|
||||
let end = start + size as usize;
|
||||
if end > heap.len() {
|
||||
return 0; // OOM
|
||||
}
|
||||
*offset = end;
|
||||
heap.as_ptr().add(start) as i32
|
||||
}
|
||||
}
|
||||
|
||||
// --- host import ---
|
||||
#[link(wasm_import_module = "swactor")]
|
||||
unsafe extern "C" {
|
||||
#[link_name = "send"]
|
||||
fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32);
|
||||
}
|
||||
|
||||
/// Message format: first 32 bytes = destination address, rest = payload.
|
||||
/// Echo sends the payload portion back to the specified destination.
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn handle(ptr: i32, len: i32) {
|
||||
if len < 32 {
|
||||
return;
|
||||
}
|
||||
let dest_ptr = ptr;
|
||||
let payload_ptr = ptr + 32;
|
||||
let payload_len = len - 32;
|
||||
unsafe {
|
||||
host_send(dest_ptr, payload_ptr, payload_len);
|
||||
}
|
||||
}
|
||||
|
||||
#[panic_handler]
|
||||
fn panic(_info: &PanicInfo) -> ! {
|
||||
loop {}
|
||||
}
|
||||
7
crates/wasm-actor/tests/guests/silent/Cargo.lock
generated
Normal file
7
crates/wasm-actor/tests/guests/silent/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "silent-guest"
|
||||
version = "0.1.0"
|
||||
13
crates/wasm-actor/tests/guests/silent/Cargo.toml
Normal file
13
crates/wasm-actor/tests/guests/silent/Cargo.toml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
[workspace]
|
||||
|
||||
[package]
|
||||
name = "silent-guest"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[profile.release]
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
45
crates/wasm-actor/tests/guests/silent/src/lib.rs
Normal file
45
crates/wasm-actor/tests/guests/silent/src/lib.rs
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
#![no_std]
|
||||
|
||||
use core::cell::UnsafeCell;
|
||||
use core::panic::PanicInfo;
|
||||
|
||||
// --- bump allocator ---
|
||||
const HEAP_SIZE: usize = 65536;
|
||||
|
||||
struct BumpAlloc {
|
||||
heap: UnsafeCell<[u8; HEAP_SIZE]>,
|
||||
offset: UnsafeCell<usize>,
|
||||
}
|
||||
|
||||
unsafe impl Sync for BumpAlloc {}
|
||||
|
||||
static ALLOC: BumpAlloc = BumpAlloc {
|
||||
heap: UnsafeCell::new([0u8; HEAP_SIZE]),
|
||||
offset: UnsafeCell::new(0),
|
||||
};
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn alloc(size: i32) -> i32 {
|
||||
unsafe {
|
||||
let offset = &mut *ALLOC.offset.get();
|
||||
let heap = &mut *ALLOC.heap.get();
|
||||
let align = 8;
|
||||
let start = (*offset + align - 1) & !(align - 1);
|
||||
let end = start + size as usize;
|
||||
if end > heap.len() {
|
||||
return 0; // OOM
|
||||
}
|
||||
*offset = end;
|
||||
heap.as_ptr().add(start) as i32
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn handle(_ptr: i32, _len: i32) {
|
||||
// Silent: receive bytes, do nothing
|
||||
}
|
||||
|
||||
#[panic_handler]
|
||||
fn panic(_info: &PanicInfo) -> ! {
|
||||
loop {}
|
||||
}
|
||||
331
crates/wasm-actor/tests/wasm_actor.rs
Normal file
331
crates/wasm-actor/tests/wasm_actor.rs
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
|
||||
use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder, WasmActorError};
|
||||
|
||||
fn guest_wasm(name: &str) -> Vec<u8> {
|
||||
let path = format!(
|
||||
"{}/tests/guests/{name}/target/wasm32-unknown-unknown/release/{name}_guest.wasm",
|
||||
env!("CARGO_MANIFEST_DIR")
|
||||
);
|
||||
std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {path}: {e}"))
|
||||
}
|
||||
|
||||
/// Build a message with an inbox address prepended (the guest contract).
|
||||
fn framed_msg(dest: &ActorAddress, payload: &[u8]) -> ByteMessage {
|
||||
let mut buf = Vec::with_capacity(32 + payload.len());
|
||||
buf.extend_from_slice(&dest.0);
|
||||
buf.extend_from_slice(payload);
|
||||
ByteMessage(buf)
|
||||
}
|
||||
|
||||
// ── Echo: send bytes in, same bytes come back ────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn echo_returns_same_payload() {
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, guest_wasm("echo"))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
let payload = b"hello wasm";
|
||||
rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap();
|
||||
rt.tick();
|
||||
|
||||
let received = inbox.try_recv().expect("inbox should have a message");
|
||||
assert_eq!(received.0, payload);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn echo_preserves_binary_payload() {
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, guest_wasm("echo"))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
let payload: Vec<u8> = (0..=255).collect();
|
||||
rt.send_to(addr, framed_msg(inbox.addr(), &payload)).unwrap();
|
||||
rt.tick();
|
||||
|
||||
let received = inbox.try_recv().expect("inbox should have a message");
|
||||
assert_eq!(received.0, payload);
|
||||
}
|
||||
|
||||
// ── Silent: processes messages without sending anything ───────────────────────
|
||||
|
||||
#[test]
|
||||
fn silent_produces_no_output() {
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, guest_wasm("silent"))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
rt.send_to(addr, ByteMessage(b"ignored".to_vec())).unwrap();
|
||||
rt.tick();
|
||||
|
||||
assert!(inbox.try_recv().is_none(), "silent guest should not send anything");
|
||||
}
|
||||
|
||||
// ── Double: one message in, two messages out ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn double_sends_two_copies() {
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, guest_wasm("double"))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
let payload = b"dup me";
|
||||
rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap();
|
||||
rt.tick();
|
||||
|
||||
let first = inbox.try_recv().expect("should receive first copy");
|
||||
let second = inbox.try_recv().expect("should receive second copy");
|
||||
assert_eq!(first.0, payload);
|
||||
assert_eq!(second.0, payload);
|
||||
assert!(inbox.try_recv().is_none(), "exactly two messages expected");
|
||||
}
|
||||
|
||||
// ── Missing export → WasmActorError::MissingExport ───────────────────────────
|
||||
|
||||
#[test]
|
||||
fn missing_alloc_export_returns_error() {
|
||||
// Minimal valid Wasm module: (module) — no exports at all
|
||||
let minimal_wasm = wat::parse_str("(module)").unwrap();
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let result = WasmActorBuilder::new(engine, minimal_wasm).build();
|
||||
match result {
|
||||
Err(WasmActorError::MissingExport(name)) => {
|
||||
assert!(
|
||||
name == "memory" || name == "alloc",
|
||||
"expected missing memory or alloc, got: {name}"
|
||||
);
|
||||
}
|
||||
Err(other) => panic!("expected MissingExport, got: {other}"),
|
||||
Ok(_) => panic!("expected error for module with no exports"),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Engine sharing: two actors from the same engine ──────────────────────────
|
||||
|
||||
#[test]
|
||||
fn shared_engine_serves_multiple_actors() {
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
|
||||
let echo = WasmActorBuilder::new(engine.clone(), guest_wasm("echo"))
|
||||
.build()
|
||||
.unwrap();
|
||||
let silent = WasmActorBuilder::new(engine, guest_wasm("silent"))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
|
||||
let echo_addr = rt.spawn(echo).unwrap();
|
||||
let _silent_addr = rt.spawn(silent).unwrap();
|
||||
|
||||
let payload = b"shared engine test";
|
||||
rt.send_to(echo_addr, framed_msg(inbox.addr(), payload)).unwrap();
|
||||
rt.tick();
|
||||
|
||||
let received = inbox.try_recv().expect("echo actor should still work");
|
||||
assert_eq!(received.0, payload);
|
||||
}
|
||||
|
||||
// ── Safety: edge cases that previously caused panics or corruption ────────────
|
||||
|
||||
#[test]
|
||||
fn oob_send_traps_cleanly_and_actor_survives() {
|
||||
// Guest calls swactor.send with dest_ptr pointing past the end of memory.
|
||||
// The host should trap the call; the actor should survive for future messages.
|
||||
let wat = r#"
|
||||
(module
|
||||
(import "swactor" "send" (func $send (param i32 i32 i32)))
|
||||
(memory (export "memory") 1)
|
||||
(func (export "alloc") (param i32) (result i32)
|
||||
i32.const 0 ;; return start of memory (simplistic)
|
||||
)
|
||||
(func (export "handle") (param i32 i32)
|
||||
;; Call send with dest_ptr = 65536 (1 page = end of memory, OOB for 32 bytes)
|
||||
i32.const 65536
|
||||
i32.const 0
|
||||
i32.const 0
|
||||
call $send
|
||||
)
|
||||
)
|
||||
"#;
|
||||
let wasm = wat::parse_str(wat).unwrap();
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
// Send a message — handle will try OOB send, which traps
|
||||
rt.send_to(addr, ByteMessage(vec![42])).unwrap();
|
||||
rt.tick();
|
||||
|
||||
// No message should arrive (the send was invalid)
|
||||
assert!(inbox.try_recv().is_none(), "OOB send should not produce a message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alloc_oom_drops_message_actor_stays_alive() {
|
||||
// Guest alloc always returns 0 (OOM). Message should be dropped,
|
||||
// actor should remain alive for subsequent messages.
|
||||
let wat = r#"
|
||||
(module
|
||||
(import "swactor" "send" (func $send (param i32 i32 i32)))
|
||||
(memory (export "memory") 1)
|
||||
(func (export "alloc") (param i32) (result i32)
|
||||
i32.const 0 ;; always OOM
|
||||
)
|
||||
(func (export "handle") (param i32 i32)
|
||||
;; Should never be called if alloc returned 0 for non-zero len
|
||||
)
|
||||
)
|
||||
"#;
|
||||
let wasm = wat::parse_str(wat).unwrap();
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
// Send a non-empty message — alloc returns 0, message should be dropped
|
||||
rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap();
|
||||
rt.tick();
|
||||
|
||||
// Actor is still alive — send another message, tick again (no panic)
|
||||
rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap();
|
||||
rt.tick();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_alloc_ptr_drops_message() {
|
||||
// Guest alloc returns -1. Host should detect the negative pointer and drop.
|
||||
let wat = r#"
|
||||
(module
|
||||
(import "swactor" "send" (func $send (param i32 i32 i32)))
|
||||
(memory (export "memory") 1)
|
||||
(func (export "alloc") (param i32) (result i32)
|
||||
i32.const -1 ;; invalid negative pointer
|
||||
)
|
||||
(func (export "handle") (param i32 i32))
|
||||
)
|
||||
"#;
|
||||
let wasm = wat::parse_str(wat).unwrap();
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
rt.send_to(addr, ByteMessage(vec![1])).unwrap();
|
||||
rt.tick(); // should not panic
|
||||
|
||||
// Actor survives
|
||||
rt.send_to(addr, ByteMessage(vec![2])).unwrap();
|
||||
rt.tick();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_trap_drops_message_actor_survives() {
|
||||
// Guest handle executes `unreachable`, causing a Wasm trap.
|
||||
// Message should be dropped, actor should stay alive.
|
||||
let wat = r#"
|
||||
(module
|
||||
(import "swactor" "send" (func $send (param i32 i32 i32)))
|
||||
(memory (export "memory") 1)
|
||||
(func (export "alloc") (param i32) (result i32)
|
||||
i32.const 256 ;; valid allocation
|
||||
)
|
||||
(func (export "handle") (param i32 i32)
|
||||
unreachable ;; trap!
|
||||
)
|
||||
)
|
||||
"#;
|
||||
let wasm = wat::parse_str(wat).unwrap();
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, wasm).build().unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
|
||||
rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap();
|
||||
rt.tick(); // handle traps, but actor should survive
|
||||
|
||||
// Actor is still alive
|
||||
rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap();
|
||||
rt.tick();
|
||||
}
|
||||
|
||||
// ── Integration: WasmActor alongside a native Rust actor ─────────────────────
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ForwardToWasm {
|
||||
wasm_addr: ActorAddress,
|
||||
inbox_addr: ActorAddress,
|
||||
}
|
||||
|
||||
struct Forwarder;
|
||||
|
||||
impl ActorInterface for Forwarder {
|
||||
type Incoming = ForwardToWasm;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, msg: ForwardToWasm) {
|
||||
// Build the framed message and forward to the wasm actor
|
||||
let payload = b"from native";
|
||||
let framed = framed_msg(&msg.inbox_addr, payload);
|
||||
let _ = ctx.send(msg.wasm_addr, framed);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_actor_communicates_with_wasm_actor() {
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
let wasm = WasmActorBuilder::new(engine, guest_wasm("echo"))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let inbox = rt.new_inbox::<ByteMessage>().unwrap();
|
||||
|
||||
let wasm_addr = rt.spawn(wasm).unwrap();
|
||||
let forwarder_addr = rt.spawn(Forwarder).unwrap();
|
||||
|
||||
rt.send_to(
|
||||
forwarder_addr,
|
||||
ForwardToWasm {
|
||||
wasm_addr,
|
||||
inbox_addr: *inbox.addr(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Tick 1: Forwarder receives message and sends to WasmActor
|
||||
rt.tick();
|
||||
// Tick 2: WasmActor receives the forwarded message and echoes to inbox
|
||||
rt.tick();
|
||||
|
||||
let received = inbox.try_recv().expect("wasm actor should have echoed");
|
||||
assert_eq!(received.0, b"from native");
|
||||
}
|
||||
191
docs/development_history/WASM_ACTOR.md
Normal file
191
docs/development_history/WASM_ACTOR.md
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
# Wasm Actor Crate — Development History
|
||||
|
||||
> Adds a new crate (`crates/wasm-actor/`) that runs WebAssembly guest code
|
||||
> **inside** a swactor actor. The Wasm instance lives in the actor — not as a
|
||||
> separate OS process. Messages arrive as bytes, get written into Wasm linear
|
||||
> memory, and the guest's `handle` export is called.
|
||||
>
|
||||
> ~350 lines of Rust (host) · 3 guest modules · 7 tests
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Overview & Motivation](#1-overview--motivation)
|
||||
2. [What Was Built](#2-what-was-built)
|
||||
3. [Guest ↔ Host Contract](#3-guest--host-contract)
|
||||
4. [Handle Cycle (Hot Path)](#4-handle-cycle-hot-path)
|
||||
5. [Guest Modules](#5-guest-modules)
|
||||
6. [Design Decisions & Tradeoffs](#6-design-decisions--tradeoffs)
|
||||
7. [Known Gaps & Future Improvements](#7-known-gaps--future-improvements)
|
||||
8. [Test Coverage Summary](#8-test-coverage-summary)
|
||||
|
||||
---
|
||||
|
||||
## 1. Overview & Motivation
|
||||
|
||||
Swactor already supported running *inside* a browser via `crates/wasm/`
|
||||
(wasm-bindgen). This crate flips the direction: run untrusted Wasm code
|
||||
*inside* an actor, sandboxed by wasmtime. Use cases include user-defined
|
||||
plugins, multi-language actors, and capability-restricted compute.
|
||||
|
||||
The main swactor crate has no wasmtime dependency — all Wasm machinery is
|
||||
isolated in `crates/wasm-actor/`.
|
||||
|
||||
---
|
||||
|
||||
## 2. What Was Built
|
||||
|
||||
| Component | Location | Purpose |
|
||||
|-----------|----------|---------|
|
||||
| `swactor-wasm-actor` crate | `crates/wasm-actor/` | Host-side: engine, builder, actor impl |
|
||||
| 3 guest crates | `crates/wasm-actor/tests/guests/{echo,double,silent}/` | `#![no_std]` Wasm modules for testing |
|
||||
| Integration tests | `crates/wasm-actor/tests/wasm_actor.rs` | 7 behavioral tests |
|
||||
|
||||
### Crate modules
|
||||
|
||||
```
|
||||
crates/wasm-actor/src/
|
||||
lib.rs — ByteMessage, re-exports
|
||||
engine.rs — SharedEngine (Arc<wasmtime::Engine>)
|
||||
builder.rs — WasmActorBuilder (compile + link + instantiate)
|
||||
actor.rs — WasmActor implementing ActorInterface
|
||||
error.rs — WasmActorError enum
|
||||
```
|
||||
|
||||
### Public types
|
||||
|
||||
- **`ByteMessage(pub Vec<u8>)`** — message type for Wasm actors. Satisfies
|
||||
`Message` bounds trivially.
|
||||
- **`SharedEngine`** — wraps `Arc<wasmtime::Engine>`. Created once, cloned
|
||||
cheaply across actors. Sandboxed config: no threads, no SIMD, no reference
|
||||
types.
|
||||
- **`WasmActorBuilder`** — takes an engine + raw `.wasm` bytes, compiles the
|
||||
module, links the `swactor.send` host import, extracts typed function handles,
|
||||
returns a `WasmActor`.
|
||||
- **`WasmActor`** — implements `ActorInterface<Incoming = ByteMessage, Response = ()>`.
|
||||
- **`WasmActorError`** — `MissingExport(&'static str)` or `Wasmtime(wasmtime::Error)`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Guest ↔ Host Contract
|
||||
|
||||
**Guest must export:**
|
||||
|
||||
| Export | Signature | Purpose |
|
||||
|--------|-----------|---------|
|
||||
| `memory` | WebAssembly linear memory | Host reads/writes message bytes here |
|
||||
| `alloc` | `(size: i32) -> i32` | Allocate `size` bytes, return pointer |
|
||||
| `handle` | `(ptr: i32, len: i32)` | Process message at `(ptr, len)` |
|
||||
|
||||
**Guest may import:**
|
||||
|
||||
| Import | Module | Signature | Purpose |
|
||||
|--------|--------|-----------|---------|
|
||||
| `send` | `swactor` | `(dest_ptr: i32, payload_ptr: i32, payload_len: i32)` | Send a message to another actor |
|
||||
|
||||
`dest_ptr` points to 32 bytes of `ActorAddress` in guest linear memory.
|
||||
`payload_ptr` + `payload_len` describe the message bytes.
|
||||
|
||||
---
|
||||
|
||||
## 4. Handle Cycle (Hot Path)
|
||||
|
||||
```
|
||||
ByteMessage arrives
|
||||
│
|
||||
v
|
||||
1. host calls guest alloc(msg.len) → ptr
|
||||
│
|
||||
v
|
||||
2. host writes msg bytes into guest memory at ptr
|
||||
│
|
||||
v
|
||||
3. host calls guest handle(ptr, len)
|
||||
│
|
||||
├── guest may call swactor.send() N times
|
||||
│ └── each appends (ActorAddress, Vec<u8>) to HostState.outbox
|
||||
│
|
||||
v
|
||||
4. host drains outbox → ctx.send(dest, ByteMessage(payload)) for each
|
||||
```
|
||||
|
||||
Traps during `alloc` or `handle` will panic. Swactor's existing
|
||||
`catch_unwind` in `tick_all` poisons the actor — consistent with the
|
||||
panic-safety model.
|
||||
|
||||
---
|
||||
|
||||
## 5. Guest Modules
|
||||
|
||||
Three `#![no_std]` Rust crates compiled to `wasm32-unknown-unknown`:
|
||||
|
||||
| Guest | Behavior | Tests it supports |
|
||||
|-------|----------|-------------------|
|
||||
| `echo` | Reads 32-byte dest + payload from message; sends payload back to dest | Echo roundtrip, binary preservation |
|
||||
| `double` | Same framing; sends payload back **twice** | Multi-send verification |
|
||||
| `silent` | Receives bytes; does nothing | No-output / no-error baseline |
|
||||
|
||||
Each guest uses a simple inline bump allocator (64 KiB heap, 8-byte aligned)
|
||||
and a `#[panic_handler]` that loops. No external dependencies.
|
||||
|
||||
Message framing convention: the first 32 bytes of the `ByteMessage` payload
|
||||
are the destination `ActorAddress`, followed by the actual message bytes.
|
||||
This allows guests to send replies without hardcoding addresses.
|
||||
|
||||
### Building guests
|
||||
|
||||
```bash
|
||||
rustup target add wasm32-unknown-unknown # one-time
|
||||
|
||||
cd crates/wasm-actor/tests/guests/echo && cargo build --target wasm32-unknown-unknown --release
|
||||
cd crates/wasm-actor/tests/guests/double && cargo build --target wasm32-unknown-unknown --release
|
||||
cd crates/wasm-actor/tests/guests/silent && cargo build --target wasm32-unknown-unknown --release
|
||||
```
|
||||
|
||||
Each guest crate has its own `[workspace]` marker to stay independent of the
|
||||
root workspace.
|
||||
|
||||
---
|
||||
|
||||
## 6. Design Decisions & Tradeoffs
|
||||
|
||||
| # | Decision | Rationale |
|
||||
|---|----------|-----------|
|
||||
| 1 | **wasmtime, not wasmer/wasm3** | Best-maintained, fuel metering support, cranelift JIT |
|
||||
| 2 | **Raw bytes, not structured messages** | Keeps the boundary simple; framing/serialization is the guest's concern |
|
||||
| 3 | **Separate crate, not a feature flag** | wasmtime is ~30 crates; most users don't need it in their dependency tree |
|
||||
| 4 | **Bump allocator in guests** | Zero-dependency, predictable, sufficient for request/response patterns |
|
||||
| 5 | **Dest address in message payload** | Avoids hardcoded addresses; guests can send to any actor the host tells them about |
|
||||
| 6 | **Traps = panics (no Result)** | Matches swactor's existing panic-safety model; `catch_unwind` in `tick_all` poisons the actor |
|
||||
| 7 | **Engine sharing via Arc** | Module compilation is expensive; `SharedEngine` amortizes it across actors |
|
||||
| 8 | **Maximum sandboxing defaults** | Disabled: threads, SIMD, relaxed SIMD, reference types, multi-value. Enabled: bulk memory (required by most compilers) |
|
||||
|
||||
---
|
||||
|
||||
## 7. Known Gaps & Future Improvements
|
||||
|
||||
| # | Gap | Notes |
|
||||
|---|-----|-------|
|
||||
| 1 | **No fuel metering** | wasmtime supports fuel; maps naturally to per-tick actor budgets. Deferred to follow-up. |
|
||||
| 2 | **No WASI** | No filesystem, network, random, or clock access. Intentional for sandboxing, but limits guest capabilities. |
|
||||
| 3 | **No guest SDK crate** | The test guests serve as examples. A published `swactor-guest` crate with the alloc/handle/send glue would reduce boilerplate. |
|
||||
| 4 | **Bump allocator never frees** | Fine for short-lived handle calls, but long-running actors would need a real allocator. |
|
||||
| 5 | **No pre-compilation cache** | `Module::new()` recompiles every time. wasmtime supports serialized modules for faster cold starts. |
|
||||
| 6 | **`cargo test -p` doesn't resolve** | Must use `--manifest-path`. Workspace resolution quirk. |
|
||||
|
||||
---
|
||||
|
||||
## 8. Test Coverage Summary
|
||||
|
||||
7 behavioral tests in `crates/wasm-actor/tests/wasm_actor.rs`:
|
||||
|
||||
| Test | Scenario |
|
||||
|------|----------|
|
||||
| `echo_returns_same_payload` | Send bytes → wasm echoes them back to inbox |
|
||||
| `echo_preserves_binary_payload` | All 256 byte values survive the roundtrip |
|
||||
| `silent_produces_no_output` | Guest does nothing; no error, no messages |
|
||||
| `double_sends_two_copies` | One message in → two messages out |
|
||||
| `missing_alloc_export_returns_error` | WAT module with no exports → `WasmActorError::MissingExport` |
|
||||
| `shared_engine_serves_multiple_actors` | Two actors from the same `SharedEngine` work independently |
|
||||
| `native_actor_communicates_with_wasm_actor` | Native Rust actor → WasmActor → inbox (two-tick delivery) |
|
||||
115
docs/wasm-actor.md
Normal file
115
docs/wasm-actor.md
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# Wasm Actor
|
||||
|
||||
The `swactor-wasm-actor` crate runs WebAssembly guest code inside a swactor
|
||||
actor. The Wasm instance is sandboxed by [wasmtime](https://wasmtime.dev/).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─ Runtime ──────────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ ┌─ WasmActor ──────────────────────────────────────────────────────┐ │
|
||||
│ │ │ │
|
||||
│ │ Store<HostState> -- wasmtime store with outbox │ │
|
||||
│ │ Memory -- guest linear memory │ │
|
||||
│ │ alloc: TypedFunc -- guest allocator │ │
|
||||
│ │ handle: TypedFunc -- guest message handler │ │
|
||||
│ │ │ │
|
||||
│ │ impl ActorInterface for WasmActor │ │
|
||||
│ │ Incoming = ByteMessage │ │
|
||||
│ │ Response = () │ │
|
||||
│ │ │ │
|
||||
│ └──────────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌─ Native Actors ─────────────────────────────────────────────────┐ │
|
||||
│ │ (can exchange ByteMessage with WasmActors normally) │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Message Flow
|
||||
|
||||
```
|
||||
Host Guest (Wasm)
|
||||
──── ────────────
|
||||
|
||||
ByteMessage arrives
|
||||
│
|
||||
├─1─ call alloc(len) ──────────► bump-allocate, return ptr
|
||||
│
|
||||
├─2─ write bytes at ptr ───────► (memory updated)
|
||||
│
|
||||
├─3─ call handle(ptr, len) ────► process message
|
||||
│ │
|
||||
│ ◄── swactor.send() ────────────┤ (0..N times)
|
||||
│ (buffered in HostState.outbox) │
|
||||
│ │
|
||||
├─4─ drain outbox ◄────────────── handle returns
|
||||
│
|
||||
v
|
||||
ctx.send(dest, ByteMessage) for each outbox entry
|
||||
```
|
||||
|
||||
## Guest Contract
|
||||
|
||||
Guests are standalone `wasm32-unknown-unknown` modules. They export three
|
||||
symbols and may import one:
|
||||
|
||||
| Direction | Module | Symbol | Signature |
|
||||
|-----------|--------|--------|-----------|
|
||||
| **export** | — | `memory` | linear memory |
|
||||
| **export** | — | `alloc` | `(i32) -> i32` |
|
||||
| **export** | — | `handle` | `(i32, i32) -> ()` |
|
||||
| **import** | `swactor` | `send` | `(i32, i32, i32) -> ()` |
|
||||
|
||||
The `send` import takes `(dest_ptr, payload_ptr, payload_len)` where
|
||||
`dest_ptr` points to a 32-byte `ActorAddress` in guest memory.
|
||||
|
||||
## Usage
|
||||
|
||||
```rust
|
||||
use swactor::runtime::{Runtime, RuntimeConfig};
|
||||
use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder};
|
||||
|
||||
// Create a shared engine (once)
|
||||
let engine = SharedEngine::new().unwrap();
|
||||
|
||||
// Build an actor from .wasm bytes
|
||||
let wasm_bytes = std::fs::read("my_guest.wasm").unwrap();
|
||||
let actor = WasmActorBuilder::new(engine, wasm_bytes)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Use it like any other actor
|
||||
let rt = Runtime::new(RuntimeConfig::default());
|
||||
let addr = rt.spawn(actor).unwrap();
|
||||
rt.send_to(addr, ByteMessage(b"hello".to_vec())).unwrap();
|
||||
rt.tick();
|
||||
```
|
||||
|
||||
## Sandboxing
|
||||
|
||||
The `SharedEngine` disables all optional Wasm proposals:
|
||||
|
||||
- Threads — disabled
|
||||
- SIMD / relaxed SIMD — disabled
|
||||
- Reference types — disabled
|
||||
- Multi-value — disabled
|
||||
- Bulk memory — **enabled** (required by most Rust/LLVM toolchains)
|
||||
|
||||
No WASI imports are linked. Guests have no access to the filesystem, network,
|
||||
clock, or random number generator. The only host function available is
|
||||
`swactor.send`.
|
||||
|
||||
## Where Things Live
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `crates/wasm-actor/src/lib.rs` | `ByteMessage` + re-exports |
|
||||
| `crates/wasm-actor/src/engine.rs` | `SharedEngine` — sandboxed wasmtime config |
|
||||
| `crates/wasm-actor/src/builder.rs` | `WasmActorBuilder` — compile, link, instantiate |
|
||||
| `crates/wasm-actor/src/actor.rs` | `WasmActor` — `ActorInterface` impl |
|
||||
| `crates/wasm-actor/src/error.rs` | `WasmActorError` |
|
||||
| `crates/wasm-actor/tests/guests/` | Three test guest crates (echo, double, silent) |
|
||||
| `crates/wasm-actor/tests/wasm_actor.rs` | 7 integration tests |
|
||||
63
src/actor.rs
63
src/actor.rs
|
|
@ -2,6 +2,31 @@ use std::any::Any;
|
|||
|
||||
use crate::Error;
|
||||
|
||||
/// Why an actor exited.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum ExitReason {
|
||||
/// Actor was explicitly stopped or removed from the pool.
|
||||
Stopped,
|
||||
/// Actor panicked during message handling.
|
||||
Panicked,
|
||||
/// The node hosting the actor left the cluster (SWIM Dead).
|
||||
NodeDown,
|
||||
}
|
||||
|
||||
/// Delivered to watchers when a watched actor exits.
|
||||
///
|
||||
/// Implements `Message` (Clone + Send + Sync + 'static) so it can be
|
||||
/// delivered through normal mailbox channels.
|
||||
#[derive(Debug, Clone)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct ActorExited {
|
||||
/// The address of the actor that died.
|
||||
pub addr: ActorAddress,
|
||||
/// Why it exited.
|
||||
pub reason: ExitReason,
|
||||
}
|
||||
|
||||
/// The primary trait defining data that can be passed to and from actor processes
|
||||
pub trait Message: 'static + Sized + Clone + Send + Sync {}
|
||||
impl<T: 'static + Sized + Clone + Send + Sync> Message for T {}
|
||||
|
|
@ -32,6 +57,11 @@ pub trait ActorInterface: 'static + Send {
|
|||
/// If your `Incoming` type IS `Down`, this method is never called — the
|
||||
/// normal `handle()` receives the message instead.
|
||||
fn handle_down(&mut self, _ctx: &Ctx, _down: Down) {}
|
||||
|
||||
/// Called when a watched actor exits. Override to react to death notifications.
|
||||
///
|
||||
/// Default: no-op (notification is silently consumed).
|
||||
fn on_actor_exit(&mut self, _ctx: &Ctx, _exited: ActorExited) {}
|
||||
}
|
||||
|
||||
/// A unique address for this actor. 32 bytes is overkill for a small application,
|
||||
|
|
@ -106,10 +136,17 @@ where
|
|||
}
|
||||
Err(msg) => msg,
|
||||
};
|
||||
match msg.downcast::<Down>() {
|
||||
let msg = match msg.downcast::<Down>() {
|
||||
Ok(down) => {
|
||||
self.inner.handle_down(ctx, *down);
|
||||
Some("swactor::actor::Down")
|
||||
return Some("swactor::actor::Down");
|
||||
}
|
||||
Err(msg) => msg,
|
||||
};
|
||||
match msg.downcast::<ActorExited>() {
|
||||
Ok(exited) => {
|
||||
self.inner.on_actor_exit(ctx, *exited);
|
||||
Some("ActorExited")
|
||||
}
|
||||
Err(_) => None,
|
||||
}
|
||||
|
|
@ -205,6 +242,10 @@ pub trait ContextInner {
|
|||
fn schedule_timer(&self, request: TimerRequest);
|
||||
/// Access the runtime extension (if installed).
|
||||
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>;
|
||||
/// Register a watch: watcher receives ActorExited when target dies.
|
||||
fn watch(&self, watcher: ActorAddress, target: ActorAddress);
|
||||
/// Cancel a watch.
|
||||
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress);
|
||||
}
|
||||
|
||||
/// Actor syscall interface — passed to `ActorInterface::handle()`.
|
||||
|
|
@ -290,4 +331,22 @@ impl<'a> Ctx<'a> {
|
|||
period,
|
||||
});
|
||||
}
|
||||
|
||||
/// Watch another actor's liveness. If the target dies, this actor
|
||||
/// receives an `ActorExited` message in its mailbox.
|
||||
///
|
||||
/// Watching an already-dead or non-existent actor delivers
|
||||
/// `ActorExited { reason: Stopped }` on the next tick.
|
||||
///
|
||||
/// Calling watch() multiple times on the same target is idempotent —
|
||||
/// only one notification is delivered.
|
||||
pub fn watch(&self, target: ActorAddress) {
|
||||
self.inner.watch(self.self_addr, target);
|
||||
}
|
||||
|
||||
/// Stop watching an actor. No notification will be delivered if the
|
||||
/// target subsequently dies.
|
||||
pub fn unwatch(&self, target: ActorAddress) {
|
||||
self.inner.unwatch(self.self_addr, target);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ use std::any::Any;
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::hash::{BuildHasher, Hasher};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, OnceLock, RwLock};
|
||||
use std::sync::{Arc, Mutex, OnceLock, RwLock};
|
||||
use std::thread::Thread;
|
||||
|
||||
use crate::actor::{ActorAddress, AnyActor, Message};
|
||||
use crate::channel::Sender;
|
||||
use crate::config::RuntimeConfig;
|
||||
use crate::stats::WorkerStats;
|
||||
use crate::worker::WatchRegistry;
|
||||
use crate::Error;
|
||||
|
||||
// ─── Identity Hasher for ActorAddress ───────────────────────────────────────
|
||||
|
|
@ -243,6 +244,7 @@ pub(crate) struct TickContext<'a> {
|
|||
pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>,
|
||||
/// Thread handles for waking parked workers on cross-worker sends.
|
||||
pub(crate) worker_threads: &'a [OnceLock<Thread>],
|
||||
pub(crate) watch_registry: Option<&'a Arc<Mutex<WatchRegistry>>>,
|
||||
#[cfg(feature = "transport")]
|
||||
pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>,
|
||||
#[cfg(feature = "transport")]
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
use std::any::Any;
|
||||
use std::cell::RefCell;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::thread::{self, JoinHandle, Thread};
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message, StopSignal, TimerRequest};
|
||||
use crate::actor::{Actor, ActorAddress, ActorExited, ActorInterface, AnyActor, ExitReason, Message, StopSignal, TimerRequest};
|
||||
use crate::channel::{Receiver, Sender};
|
||||
// Re-export config types so existing code using `runtime::RuntimeConfig` still works
|
||||
pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig};
|
||||
|
|
@ -14,7 +14,7 @@ use crate::extension::RuntimeExtension;
|
|||
use crate::stats::{StatsHook, WorkerStats};
|
||||
// Re-export stats types so existing code using `runtime::*` still works
|
||||
pub use crate::stats::{RuntimeStats, WorkerInfo};
|
||||
use crate::worker::Worker;
|
||||
use crate::worker::{WatchRegistry, Worker};
|
||||
use crate::Error;
|
||||
|
||||
/// Generic message inbox for receiving messages outside of the runtime.
|
||||
|
|
@ -103,6 +103,7 @@ pub struct Runtime {
|
|||
is_running: AtomicBool,
|
||||
worker_stats: Vec<Arc<WorkerStats>>,
|
||||
stats_hook: Option<Arc<dyn StatsHook>>,
|
||||
watch_registry: Arc<Mutex<WatchRegistry>>,
|
||||
/// Workers available for tick(). run() drains this and moves workers to threads.
|
||||
tick_workers: RefCell<Vec<Worker>>,
|
||||
/// Thread handles for waking parked workers. Set by workers on startup via OnceLock.
|
||||
|
|
@ -198,6 +199,7 @@ impl Runtime {
|
|||
is_running: AtomicBool::new(false),
|
||||
worker_stats,
|
||||
stats_hook: None,
|
||||
watch_registry: Arc::new(Mutex::new(WatchRegistry::new())),
|
||||
tick_workers: RefCell::new(workers),
|
||||
worker_threads,
|
||||
created_at: Instant::now(),
|
||||
|
|
@ -304,6 +306,7 @@ impl Runtime {
|
|||
extension: self.extension.as_deref(),
|
||||
stats_hook: self.stats_hook.as_deref(),
|
||||
worker_threads: &self.worker_threads,
|
||||
watch_registry: Some(&self.watch_registry),
|
||||
#[cfg(feature = "transport")]
|
||||
codec_registry: self.codec_registry.as_deref(),
|
||||
#[cfg(feature = "transport")]
|
||||
|
|
@ -506,4 +509,25 @@ impl ContextInner for Runtime {
|
|||
fn extension(&self) -> Option<&dyn RuntimeExtension> {
|
||||
self.extension.as_deref()
|
||||
}
|
||||
|
||||
fn watch(&self, watcher: ActorAddress, target: ActorAddress) {
|
||||
if self.address_map.lookup(&target).is_some() {
|
||||
self.watch_registry.lock().unwrap().watch(watcher, target);
|
||||
} else {
|
||||
// Target not found — deliver ActorExited immediately.
|
||||
let msg = ActorExited {
|
||||
addr: target,
|
||||
reason: ExitReason::Stopped,
|
||||
};
|
||||
// Route to watcher via transfer queue
|
||||
if let Some(wid) = self.address_map.lookup(&watcher) {
|
||||
self.transfer_txs[wid.as_usize()]
|
||||
.send(Envelope::new(watcher, Box::new(msg)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) {
|
||||
self.watch_registry.lock().unwrap().unwatch(watcher, target);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
165
src/worker.rs
165
src/worker.rs
|
|
@ -1,12 +1,12 @@
|
|||
use std::any::Any;
|
||||
use std::cell::RefCell;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::actor::{ActorAddress, AnyActor, CloneMsg, ContextInner, Ctx, StopReason, StopSignal, TimerRequest};
|
||||
use crate::actor::{ActorAddress, ActorExited, AnyActor, CloneMsg, ContextInner, Ctx, ExitReason, StopReason, StopSignal, TimerRequest};
|
||||
use crate::channel::Receiver;
|
||||
use crate::config::MailboxOverflow;
|
||||
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
|
||||
|
|
@ -108,6 +108,95 @@ impl TimerWheel {
|
|||
}
|
||||
}
|
||||
|
||||
// ─── Watch Registry ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Tracks watch relationships between actors.
|
||||
///
|
||||
/// Shared across workers via `Arc<Mutex<_>>`. Contention is negligible
|
||||
/// because watch/unwatch operations are rare relative to message sends.
|
||||
pub(crate) struct WatchRegistry {
|
||||
/// target → set of watchers awaiting death notification
|
||||
watchers: HashMap<ActorAddress, HashSet<ActorAddress>>,
|
||||
/// watcher → set of targets it's watching (reverse index for cleanup)
|
||||
watching: HashMap<ActorAddress, HashSet<ActorAddress>>,
|
||||
}
|
||||
|
||||
impl WatchRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
watchers: HashMap::new(),
|
||||
watching: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) {
|
||||
self.watchers.entry(target).or_default().insert(watcher);
|
||||
self.watching.entry(watcher).or_default().insert(target);
|
||||
}
|
||||
|
||||
pub fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) {
|
||||
if let Some(set) = self.watchers.get_mut(&target) {
|
||||
set.remove(&watcher);
|
||||
if set.is_empty() {
|
||||
self.watchers.remove(&target);
|
||||
}
|
||||
}
|
||||
if let Some(set) = self.watching.get_mut(&watcher) {
|
||||
set.remove(&target);
|
||||
if set.is_empty() {
|
||||
self.watching.remove(&watcher);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs.
|
||||
pub fn notify_death(
|
||||
&mut self,
|
||||
target: ActorAddress,
|
||||
reason: ExitReason,
|
||||
) -> Vec<(ActorAddress, ActorExited)> {
|
||||
let notification = ActorExited {
|
||||
addr: target,
|
||||
reason,
|
||||
};
|
||||
let mut result = Vec::new();
|
||||
|
||||
if let Some(watcher_set) = self.watchers.remove(&target) {
|
||||
for watcher in &watcher_set {
|
||||
result.push((*watcher, notification.clone()));
|
||||
// clean up reverse index
|
||||
if let Some(set) = self.watching.get_mut(watcher) {
|
||||
set.remove(&target);
|
||||
if set.is_empty() {
|
||||
self.watching.remove(watcher);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Called when a watcher itself dies. Cleans up all its watching entries.
|
||||
pub fn cleanup_watcher(&mut self, watcher: &ActorAddress) {
|
||||
if let Some(targets) = self.watching.remove(watcher) {
|
||||
for target in targets {
|
||||
if let Some(set) = self.watchers.get_mut(&target) {
|
||||
set.remove(watcher);
|
||||
if set.is_empty() {
|
||||
self.watchers.remove(&target);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a target has any watchers registered.
|
||||
pub fn has_watchers(&self, target: &ActorAddress) -> bool {
|
||||
self.watchers.get(target).is_some_and(|s| !s.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Worker ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// A worker owns a set of actors and runs them in a loop.
|
||||
|
|
@ -203,6 +292,7 @@ impl Worker {
|
|||
let timer_requests: RefCell<Vec<TimerRequest>> = RefCell::new(Vec::new());
|
||||
|
||||
let processed;
|
||||
let deaths;
|
||||
{
|
||||
let worker_ctx = WorkerContext {
|
||||
worker_id: self.id,
|
||||
|
|
@ -212,7 +302,7 @@ impl Worker {
|
|||
timer_requests: &timer_requests,
|
||||
stats: &self.stats,
|
||||
};
|
||||
processed = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests);
|
||||
(processed, deaths) = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests);
|
||||
if processed > 0 {
|
||||
did_work = true;
|
||||
}
|
||||
|
|
@ -256,6 +346,40 @@ impl Worker {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5b. Process actor deaths → deliver ActorExited to watchers
|
||||
if !deaths.is_empty() {
|
||||
did_work = true;
|
||||
if let Some(registry) = &tc.watch_registry {
|
||||
let mut reg = registry.lock().unwrap();
|
||||
for (dead_addr, reason) in deaths {
|
||||
let notifications = reg.notify_death(dead_addr, reason);
|
||||
for (watcher_addr, msg) in notifications {
|
||||
// Deliver ActorExited as a normal message via the address map
|
||||
match tc.address_map.lookup(&watcher_addr) {
|
||||
Some(wid) if wid == self.id => {
|
||||
self.pool.deliver(&watcher_addr, Box::new(msg));
|
||||
}
|
||||
Some(wid) => {
|
||||
tc.transfer_txs[wid.as_usize()]
|
||||
.send(Envelope::new(watcher_addr, Box::new(msg)));
|
||||
}
|
||||
None => {
|
||||
// Watcher not in address map — may be an inbox or remote.
|
||||
// Try inbox registry as best effort.
|
||||
let _ = tc.inbox_registry.try_deliver(
|
||||
watcher_addr,
|
||||
Box::new(msg),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Clean up the dead actor's own watches (things it was watching)
|
||||
reg.cleanup_watcher(&dead_addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let t5 = Instant::now();
|
||||
|
||||
// 6. Publish stats (skip entirely when idle to avoid allocation + mutex)
|
||||
|
|
@ -451,6 +575,29 @@ impl ContextInner for WorkerContext<'_> {
|
|||
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
|
||||
self.tc.extension
|
||||
}
|
||||
|
||||
fn watch(&self, watcher: ActorAddress, target: ActorAddress) {
|
||||
if let Some(registry) = &self.tc.watch_registry {
|
||||
// Check if target exists in the address map
|
||||
if self.tc.address_map.lookup(&target).is_some() {
|
||||
registry.lock().unwrap().watch(watcher, target);
|
||||
} else {
|
||||
// Target not found — deliver ActorExited { reason: Stopped } immediately.
|
||||
// Buffer in pending_local so it arrives on next tick.
|
||||
let msg = ActorExited {
|
||||
addr: target,
|
||||
reason: ExitReason::Stopped,
|
||||
};
|
||||
self.pending_local.borrow_mut().push((watcher, Box::new(msg)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) {
|
||||
if let Some(registry) = &self.tc.watch_registry {
|
||||
registry.lock().unwrap().unwatch(watcher, target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ActorSlot {
|
||||
|
|
@ -531,7 +678,7 @@ impl ActorPool {
|
|||
std::mem::replace(&mut self.drops_this_tick, 0)
|
||||
}
|
||||
|
||||
/// Tick all actors in the pool. Returns the number of messages processed.
|
||||
/// Tick all actors in the pool. Returns (messages_processed, newly_dead_actors).
|
||||
///
|
||||
/// Each actor processes up to `budget` messages per tick (0 = unlimited).
|
||||
/// This prevents a single hot actor from starving others on the same worker.
|
||||
|
|
@ -541,8 +688,9 @@ impl ActorPool {
|
|||
stats: &WorkerStats,
|
||||
budget: usize,
|
||||
stop_requests: &RefCell<Vec<ActorAddress>>,
|
||||
) -> usize {
|
||||
) -> (usize, Vec<(ActorAddress, ExitReason)>) {
|
||||
let mut count = 0;
|
||||
let mut deaths = Vec::new();
|
||||
for (&addr, slot) in self.actors.iter_mut() {
|
||||
if slot.poisoned || slot.stopping {
|
||||
// Discard all messages for poisoned/stopping actors
|
||||
|
|
@ -606,6 +754,8 @@ impl ActorPool {
|
|||
#[cfg(feature = "tracing")]
|
||||
tracing::error!(actor_addr = %addr, "actor.panicked");
|
||||
slot.poisoned = true;
|
||||
slot.mailbox.clear();
|
||||
deaths.push((addr, ExitReason::Panicked));
|
||||
break;
|
||||
}
|
||||
Ok(Some(type_name)) => {
|
||||
|
|
@ -624,6 +774,7 @@ impl ActorPool {
|
|||
slot.stopping = true;
|
||||
stats.stops.fetch_add(1, Ordering::Relaxed);
|
||||
slot.mailbox.clear();
|
||||
deaths.push((addr, ExitReason::Stopped));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -633,7 +784,7 @@ impl ActorPool {
|
|||
}
|
||||
}
|
||||
}
|
||||
count
|
||||
(count, deaths)
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
|
|
|
|||
378
tests/watch_api.rs
Normal file
378
tests/watch_api.rs
Normal file
|
|
@ -0,0 +1,378 @@
|
|||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use swactor::actor::{ActorAddress, ActorExited, ActorInterface, ExitReason};
|
||||
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
|
||||
|
||||
// ── Actors ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// An actor that panics when it receives PanicMsg.
|
||||
struct PanicOnCommand;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PanicMsg;
|
||||
|
||||
impl ActorInterface for PanicOnCommand {
|
||||
type Incoming = PanicMsg;
|
||||
type Response = ();
|
||||
fn handle(&mut self, _ctx: &Ctx, _msg: PanicMsg) {
|
||||
panic!("deliberate panic for test");
|
||||
}
|
||||
}
|
||||
|
||||
/// An actor that watches targets and counts exit notifications.
|
||||
struct ExitWatcher {
|
||||
exit_count: Arc<AtomicUsize>,
|
||||
last_reason: Arc<std::sync::Mutex<Option<ExitReason>>>,
|
||||
last_addr: Arc<std::sync::Mutex<Option<ActorAddress>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum WatcherCmd {
|
||||
WatchThis(ActorAddress),
|
||||
UnwatchThis(ActorAddress),
|
||||
}
|
||||
|
||||
impl ActorInterface for ExitWatcher {
|
||||
type Incoming = WatcherCmd;
|
||||
type Response = ();
|
||||
fn handle(&mut self, ctx: &Ctx, msg: WatcherCmd) {
|
||||
match msg {
|
||||
WatcherCmd::WatchThis(target) => {
|
||||
ctx.watch(target);
|
||||
}
|
||||
WatcherCmd::UnwatchThis(target) => {
|
||||
ctx.unwatch(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn on_actor_exit(&mut self, _ctx: &Ctx, exited: ActorExited) {
|
||||
self.exit_count.fetch_add(1, Ordering::SeqCst);
|
||||
*self.last_reason.lock().unwrap() = Some(exited.reason);
|
||||
*self.last_addr.lock().unwrap() = Some(exited.addr);
|
||||
}
|
||||
}
|
||||
|
||||
impl ExitWatcher {
|
||||
fn new() -> (Self, WatcherState) {
|
||||
let exit_count = Arc::new(AtomicUsize::new(0));
|
||||
let last_reason = Arc::new(std::sync::Mutex::new(None));
|
||||
let last_addr = Arc::new(std::sync::Mutex::new(None));
|
||||
let state = WatcherState {
|
||||
exit_count: exit_count.clone(),
|
||||
last_reason: last_reason.clone(),
|
||||
last_addr: last_addr.clone(),
|
||||
};
|
||||
(
|
||||
ExitWatcher {
|
||||
exit_count,
|
||||
last_reason,
|
||||
last_addr,
|
||||
},
|
||||
state,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared state for inspecting what ExitWatcher observed.
|
||||
struct WatcherState {
|
||||
exit_count: Arc<AtomicUsize>,
|
||||
last_reason: Arc<std::sync::Mutex<Option<ExitReason>>>,
|
||||
last_addr: Arc<std::sync::Mutex<Option<ActorAddress>>>,
|
||||
}
|
||||
|
||||
impl WatcherState {
|
||||
fn count(&self) -> usize {
|
||||
self.exit_count.load(Ordering::SeqCst)
|
||||
}
|
||||
fn last_reason(&self) -> Option<ExitReason> {
|
||||
self.last_reason.lock().unwrap().clone()
|
||||
}
|
||||
fn last_addr(&self) -> Option<ActorAddress> {
|
||||
*self.last_addr.lock().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// A silent actor that does nothing (for targets that shouldn't panic).
|
||||
struct Sleeper;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Noop;
|
||||
|
||||
impl ActorInterface for Sleeper {
|
||||
type Incoming = Noop;
|
||||
type Response = ();
|
||||
fn handle(&mut self, _ctx: &Ctx, _msg: Noop) {}
|
||||
}
|
||||
|
||||
// ── Helper ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn tick_n(rt: &Runtime, n: usize) {
|
||||
for _ in 0..n {
|
||||
rt.tick();
|
||||
}
|
||||
}
|
||||
|
||||
fn single_thread_config() -> RuntimeConfig {
|
||||
RuntimeConfig {
|
||||
num_threads: 1,
|
||||
..RuntimeConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Given a watcher and a target actor,
|
||||
/// when the target panics,
|
||||
/// then the watcher's on_actor_exit fires with ExitReason::Panicked.
|
||||
#[test]
|
||||
fn watch_receives_notification_on_panic() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, state) = ExitWatcher::new();
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
|
||||
// Tell watcher to watch the target
|
||||
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
// Kill the target
|
||||
rt.send_to(target, PanicMsg).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(state.count(), 1, "watcher should have received exactly one ActorExited");
|
||||
assert_eq!(state.last_reason(), Some(ExitReason::Panicked));
|
||||
assert_eq!(state.last_addr(), Some(target));
|
||||
}
|
||||
|
||||
/// Given a watcher that watches then unwatches a target,
|
||||
/// when the target panics,
|
||||
/// then the watcher receives NO notification.
|
||||
#[test]
|
||||
fn unwatch_prevents_notification() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, state) = ExitWatcher::new();
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
|
||||
// Watch
|
||||
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
// Unwatch
|
||||
rt.send_to(watcher, WatcherCmd::UnwatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
// Kill target
|
||||
rt.send_to(target, PanicMsg).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(state.count(), 0, "after unwatch, no notification should be delivered");
|
||||
}
|
||||
|
||||
/// Given a watch on an address that was never spawned,
|
||||
/// then the watcher receives ActorExited { reason: Stopped }.
|
||||
#[test]
|
||||
fn watch_nonexistent_actor_delivers_stopped() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, state) = ExitWatcher::new();
|
||||
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
|
||||
let nonexistent = ActorAddress::new_random();
|
||||
rt.send_to(watcher, WatcherCmd::WatchThis(nonexistent)).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(state.count(), 1, "should receive ActorExited for non-existent target");
|
||||
assert_eq!(state.last_reason(), Some(ExitReason::Stopped));
|
||||
assert_eq!(state.last_addr(), Some(nonexistent));
|
||||
}
|
||||
|
||||
/// Given a watcher that dies before the target,
|
||||
/// when the target subsequently panics,
|
||||
/// then there is no panic or leak.
|
||||
#[test]
|
||||
fn watcher_dies_before_target_no_panic() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let (watcher_actor, _state) = ExitWatcher::new();
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
|
||||
// Watch
|
||||
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
// Kill the watcher first (send it a type-mismatched panic msg directly)
|
||||
// Actually, ExitWatcher doesn't panic. Use Runtime-level watch + PanicOnCommand.
|
||||
let rt2 = Runtime::new(single_thread_config());
|
||||
let target2 = rt2.spawn(PanicOnCommand).unwrap();
|
||||
let watcher2 = rt2.spawn(PanicOnCommand).unwrap();
|
||||
|
||||
use swactor::actor::ContextInner;
|
||||
rt2.watch(watcher2, target2);
|
||||
tick_n(&rt2, 3);
|
||||
|
||||
// Kill watcher first
|
||||
rt2.send_to(watcher2, PanicMsg).unwrap();
|
||||
tick_n(&rt2, 5);
|
||||
|
||||
// Kill target — should not crash
|
||||
rt2.send_to(target2, PanicMsg).unwrap();
|
||||
tick_n(&rt2, 5);
|
||||
|
||||
// If we got here, no crash.
|
||||
}
|
||||
|
||||
/// Given a watcher that calls watch() twice on the same target,
|
||||
/// when the target panics,
|
||||
/// then the watcher receives exactly one notification.
|
||||
#[test]
|
||||
fn idempotent_watch_delivers_one_notification() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, state) = ExitWatcher::new();
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
|
||||
// Watch twice
|
||||
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
// Kill target
|
||||
rt.send_to(target, PanicMsg).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(state.count(), 1, "double watch should produce exactly one notification");
|
||||
}
|
||||
|
||||
/// Given multiple watchers on the same target,
|
||||
/// when the target panics,
|
||||
/// then all watchers receive the notification.
|
||||
#[test]
|
||||
fn multiple_watchers_all_notified() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (w1_actor, s1) = ExitWatcher::new();
|
||||
let (w2_actor, s2) = ExitWatcher::new();
|
||||
let (w3_actor, s3) = ExitWatcher::new();
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let w1 = rt.spawn(w1_actor).unwrap();
|
||||
let w2 = rt.spawn(w2_actor).unwrap();
|
||||
let w3 = rt.spawn(w3_actor).unwrap();
|
||||
|
||||
rt.send_to(w1, WatcherCmd::WatchThis(target)).unwrap();
|
||||
rt.send_to(w2, WatcherCmd::WatchThis(target)).unwrap();
|
||||
rt.send_to(w3, WatcherCmd::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
rt.send_to(target, PanicMsg).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(s1.count(), 1, "watcher 1 should be notified");
|
||||
assert_eq!(s2.count(), 1, "watcher 2 should be notified");
|
||||
assert_eq!(s3.count(), 1, "watcher 3 should be notified");
|
||||
}
|
||||
|
||||
/// Self-watch doesn't crash the runtime.
|
||||
#[test]
|
||||
fn self_watch_does_not_crash() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, _state) = ExitWatcher::new();
|
||||
|
||||
let actor = rt.spawn(watcher_actor).unwrap();
|
||||
rt.send_to(actor, WatcherCmd::WatchThis(actor)).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
// No crash = pass
|
||||
}
|
||||
|
||||
/// Runtime-level watch (outside actor context) delivers notification.
|
||||
#[test]
|
||||
fn runtime_level_watch_delivers_notification() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, state) = ExitWatcher::new();
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
tick_n(&rt, 2); // ensure both spawned
|
||||
|
||||
use swactor::actor::ContextInner;
|
||||
rt.watch(watcher, target);
|
||||
|
||||
rt.send_to(target, PanicMsg).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(state.count(), 1, "runtime-level watch should deliver notification");
|
||||
assert_eq!(state.last_reason(), Some(ExitReason::Panicked));
|
||||
}
|
||||
|
||||
/// Runtime-level watch on non-existent address delivers Stopped.
|
||||
#[test]
|
||||
fn runtime_level_watch_nonexistent_delivers_stopped() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let (watcher_actor, state) = ExitWatcher::new();
|
||||
|
||||
let watcher = rt.spawn(watcher_actor).unwrap();
|
||||
tick_n(&rt, 2);
|
||||
|
||||
let fake = ActorAddress::new_random();
|
||||
use swactor::actor::ContextInner;
|
||||
rt.watch(watcher, fake);
|
||||
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(state.count(), 1, "watching non-existent from runtime should deliver Stopped");
|
||||
assert_eq!(state.last_reason(), Some(ExitReason::Stopped));
|
||||
}
|
||||
|
||||
/// Given a watcher watching target via on_actor_exit,
|
||||
/// when target panics,
|
||||
/// then the watcher can react by spawning a replacement (supervision pattern).
|
||||
#[test]
|
||||
fn watcher_can_react_to_death_by_spawning() {
|
||||
let rt = Runtime::new(single_thread_config());
|
||||
let spawned = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
struct Supervisor {
|
||||
spawned_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum SupervisorMsg {
|
||||
WatchThis(ActorAddress),
|
||||
}
|
||||
|
||||
impl ActorInterface for Supervisor {
|
||||
type Incoming = SupervisorMsg;
|
||||
type Response = ();
|
||||
fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) {
|
||||
match msg {
|
||||
SupervisorMsg::WatchThis(target) => ctx.watch(target),
|
||||
}
|
||||
}
|
||||
|
||||
fn on_actor_exit(&mut self, ctx: &Ctx, _exited: ActorExited) {
|
||||
// React: spawn a replacement
|
||||
let replacement = ctx.spawn(Sleeper).unwrap();
|
||||
let _ = replacement;
|
||||
self.spawned_count.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
let target = rt.spawn(PanicOnCommand).unwrap();
|
||||
let sup = rt.spawn(Supervisor { spawned_count: spawned.clone() }).unwrap();
|
||||
|
||||
rt.send_to(sup, SupervisorMsg::WatchThis(target)).unwrap();
|
||||
tick_n(&rt, 3);
|
||||
|
||||
rt.send_to(target, PanicMsg).unwrap();
|
||||
tick_n(&rt, 5);
|
||||
|
||||
assert_eq!(spawned.load(Ordering::SeqCst), 1, "supervisor should have spawned a replacement");
|
||||
}
|
||||
Loading…
Reference in a new issue