swactor/src/worker.rs

1341 lines
48 KiB
Rust
Raw Normal View History

use crate::Instant;
use std::any::Any;
use std::cell::RefCell;
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::sync::atomic::Ordering;
use crate::Error;
use crate::actor::{
ActorAddress, AnyActor, ContextInner, Ctx, Environment, ExitValue, ResumeSignal, SpawnRequest,
StopReason, StopSignal, StopWithSignal, SystemInfo,
};
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
use crate::admin::{
ActorStatus, ActorSummary, AdminCommand, AdminError, AdminResult, InspectActorResponse,
ListActorsResponse, OperationResult,
};
use crate::channel::Receiver;
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
use crate::stats::{ActorSnapshot, TickTiming, WorkerStats};
use crate::extension::WorkerExtension;
use crate::runtime::RuntimeShared;
/// Whether an actor should be skipped during `tick_all`.
pub(crate) fn should_skip_actor(poisoned: bool, stopping: bool, suspended: bool) -> bool {
poisoned || stopping || suspended
}
/// Whether `on_stop` should fire for an actor being cleaned up.
pub(crate) fn is_on_stop_eligible(stopping: bool, poisoned: bool) -> bool {
stopping && !poisoned
}
/// Determine the `StopReason` for a dead actor based on its flags.
pub(crate) fn determine_stop_reason(poisoned: bool, has_exit_value: bool) -> StopReason {
if poisoned {
StopReason::Panicked
} else if has_exit_value {
StopReason::Completed
} else {
StopReason::Normal
}
}
/// Route a runtime-injected message (extension output, death notification) to
/// its destination: same-worker pool first, then the owning worker's transfer
/// queue, then the non-local (inbox/transport) seam.
fn route_runtime_message(
pool: &mut ActorPool,
tc: &TickContext,
dest: ActorAddress,
msg: Box<dyn Any + Send>,
) {
if pool.contains(&dest) {
pool.deliver(&dest, msg);
} else {
match tc.worker_of(&dest) {
Some(w) => tc.transfer_tx(w).send(Envelope::new(dest, msg)),
None => {
let _ = tc.route_nonlocal(dest, msg);
}
}
}
}
/// A worker owns a disjoint set of actors and processes them via [`Worker::try_tick`].
///
/// Each worker is owned by exactly one execution host (a [`SingleThreadRuntime`]
/// or an engine driver) and requires only `Send`, not `Sync`.
pub struct Worker {
pub(crate) id: WorkerId,
pub(crate) shared: Arc<RuntimeShared>,
pub(crate) pool: ActorPool,
transfer_rx: Receiver<Envelope>,
spawn_rx: Receiver<SpawnRequest>,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
admin_rx: Receiver<AdminCommand>,
pub(crate) stats: Arc<WorkerStats>,
/// Reusable scratch buffer for building per-actor snapshots.
snapshot_buf: Vec<ActorSnapshot>,
/// Per-worker extension (e.g., timer wheel). Created by RuntimeExtension factory.
pub(crate) worker_ext: Option<Box<dyn WorkerExtension>>,
/// True if the previous tick did work — ensures one full tick follows a productive
/// tick so pending_local messages delivered to mailboxes get drained.
has_backlog: bool,
/// Transfers whose address is mapped to this worker but whose spawn request
/// has not reached the pool yet.
deferred_transfers: VecDeque<Envelope>,
/// Targeted admin commands waiting for their mapped spawn to be installed.
deferred_admin: VecDeque<AdminCommand>,
}
impl Worker {
pub(crate) fn new(
id: WorkerId,
shared: Arc<RuntimeShared>,
transfer_rx: Receiver<Envelope>,
spawn_rx: Receiver<SpawnRequest>,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
admin_rx: Receiver<AdminCommand>,
stats: Arc<WorkerStats>,
) -> Self {
Self {
id,
shared,
pool: ActorPool::new(),
transfer_rx,
spawn_rx,
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
admin_rx,
stats,
snapshot_buf: Vec::new(),
worker_ext: None,
has_backlog: false,
deferred_transfers: VecDeque::new(),
deferred_admin: VecDeque::new(),
}
}
pub(crate) fn has_work(&self) -> bool {
self.has_backlog
|| !self.spawn_rx.is_empty()
|| !self.transfer_rx.is_empty()
|| !self.deferred_transfers.is_empty()
|| !self.admin_rx.is_empty()
|| !self.deferred_admin.is_empty()
|| self
.worker_ext
.as_ref()
.map_or(false, |e| e.has_pending_work())
}
/// Run one synchronous worker pass. Returns `true` if any work was done.
///
/// This is the only core worker transition. The owning host calls it; the
/// worker never drives itself.
pub fn try_tick(&mut self) -> bool {
let wid = self.id;
// Fast idle path: skip the entire tick when nothing could have changed.
// Cost: ~3 atomic loads, zero syscalls, zero actor iteration.
if !self.has_work() {
return false;
}
// Build the routing context from disjoint fields so the mutable
// per-pass state (pool, queues, extension) can still be borrowed below.
let shared = &self.shared;
let tc = TickContext {
address_map: &shared.address_map,
spawn_txs: &shared.spawn_txs,
transfer_txs: &shared.transfer_txs,
inbox_registry: &shared.inbox_registry,
config: &shared.config,
extension: shared.extension.get().map(|a| a.as_ref()),
process_output_observer: shared.process_output_observer.get(),
stats_hook: shared.stats_hook.get().map(|a| a.as_ref()),
worker_stats: &self.stats,
num_workers: shared.worker_stats.len(),
worker_id: wid,
created_at: shared.created_at,
#[cfg(feature = "transport")]
remote_sink: shared.remote_sink.get().map(|a| a.as_ref()),
};
#[cfg(feature = "tracing")]
let _span = tracing::trace_span!("worker.tick", worker_id = wid.index()).entered();
let mut did_work = false;
let t0 = Instant::now();
// 1. Drain spawn queue → add actors to pool
did_work |= Self::drain_spawns(
&mut self.pool,
&self.spawn_rx,
&tc,
tc.config.worker_ingress_budget,
);
let t1 = Instant::now();
// 2. Drain retained transfers first, then the transfer queue → deliver
// envelopes to actors or retain them until their mapped spawn installs.
did_work |= Self::drain_transfers(
&mut self.pool,
&self.transfer_rx,
&mut self.deferred_transfers,
&tc,
tc.config.worker_ingress_budget,
);
let t2 = Instant::now();
// 3. Drain admin queue → inspect or mutate worker-owned slots before handlers
did_work |= Self::drain_admin(
&mut self.pool,
&self.admin_rx,
&mut self.deferred_admin,
&tc,
tc.config.worker_ingress_budget,
);
// 4. Fire per-worker extension (e.g., timers) → deliver before tick_all
let ext_msgs: Vec<_> = self
.worker_ext
.as_mut()
.map(|ext| ext.on_tick())
.unwrap_or_default();
for (dest, msg) in ext_msgs {
route_runtime_message(&mut self.pool, &tc, dest, msg);
did_work = true;
}
// 5. Tick all actors with WorkerContext
let pending_local: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
RefCell::new(Vec::new());
let stop_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let stop_with_values: RefCell<Vec<(ActorAddress, ExitValue)>> = RefCell::new(Vec::new());
let suspend_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let worker_requests: RefCell<Vec<Box<dyn Any + Send>>> = RefCell::new(Vec::new());
let processed;
{
let worker_ctx = WorkerContext {
tc: &tc,
pending_local: &pending_local,
stop_requests: &stop_requests,
stop_with_values: &stop_with_values,
suspend_requests: &suspend_requests,
worker_requests: &worker_requests,
stats: &self.stats,
};
processed =
self.pool
.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget);
if processed > 0 {
did_work = true;
}
}
let t3 = Instant::now();
#[cfg(feature = "tracing")]
if processed > 0 {
tracing::debug!(
worker_id = wid.index(),
messages_processed = processed,
"worker.tick_all"
);
}
// 6. Drain spawn queue again — actors spawned during step 5
// must be in the pool before pending_local delivery.
did_work |= Self::drain_spawns(
&mut self.pool,
&self.spawn_rx,
&tc,
tc.config.worker_ingress_budget,
);
let t4 = Instant::now();
// 7. Drain pending_local buffer → deliver to local actors
let pending = pending_local.into_inner();
if !pending.is_empty() {
did_work = true;
}
for (addr, msg) in pending {
Self::deliver_or_defer_transfer(
&mut self.pool,
&tc,
&mut self.deferred_transfers,
Envelope::new(addr, msg),
);
}
// 7.5. Process worker extension requests from handlers (e.g., timer scheduling)
if let Some(ext) = &mut self.worker_ext {
for request in worker_requests.into_inner() {
ext.handle_request(request);
}
}
let t5 = Instant::now();
// 8. Publish stats (skip entirely when idle to avoid allocation + mutex)
if did_work {
self.stats
.num_actors
.store(self.pool.len(), Ordering::Relaxed);
self.stats
.total_mailbox_depth
.store(self.pool.total_mailbox_depth(), Ordering::Relaxed);
self.stats
.messages_processed
.fetch_add(processed as u64, Ordering::Relaxed);
if let Some(hook) = tc.stats_hook {
self.pool.mailbox_depths_into(&mut self.snapshot_buf);
hook.on_tick(wid.index(), &self.snapshot_buf);
}
}
let t6 = Instant::now();
// Record tick timing
let timing = TickTiming {
phase_us: [
t1.duration_since(t0).as_micros() as u64,
t2.duration_since(t1).as_micros() as u64,
t3.duration_since(t2).as_micros() as u64,
t4.duration_since(t3).as_micros() as u64,
t5.duration_since(t4).as_micros() as u64,
t6.duration_since(t5).as_micros() as u64,
],
messages_processed: processed,
did_work,
};
self.stats.push_tick_timing(timing);
#[cfg(feature = "tracing")]
if did_work {
tracing::debug!(
worker_id = wid.index(),
num_actors = self.pool.len(),
mailbox_depth = self.pool.total_mailbox_depth(),
messages_processed = processed,
"worker.stats"
);
}
feat(myelin): enforce actor-owned control flow Architecture enforcement: - Install a repository-owned rustc wrapper for ordinary cargo check, build, and test commands. Resolve compiler item identities so renamed imports and helper wrappers cannot hide spawning, timing, blocking, polling, thread, or runtime-driving capabilities. - Define the execution-owner crates and reject dependencies from those substrates back into Myelin policy. Add compile-pass and compile-fail contracts for actor helpers, execution owners, test waits, forbidden capabilities, suppression attempts, and owner dependency inversions. Execution ownership: - Add engine-owned actor timers with cancellation and generation identity, then migrate lifecycle deadlines and protocol ticks off application tasks. Keep networking, process output, telemetry, and blocking provider calls in their approved I/O substrates. - Move process spawn, wait, signal, Unix listener, and output-following mechanics into swactor-process. Isolate Vast.ai blocking HTTP mechanics behind its adapter while actors retain retry, recovery, and provisioning decisions. Myelin control flow: - Rework manual control, worker lifecycle, provisioning, provider recovery, job deployment, distribution, edge orchestration, and shutdown as actor state transitions and typed effects. Preserve durable provider adoption and command outcomes across graceful and abrupt restarts. - Replace controller loops and timer-forwarding tasks with actor messages; leave substrate tasks as cancellable observation streams with no durable policy state. Properties and resource ownership: - Add deterministic engine and component properties, a stateful mock-VastAI lifecycle model, persisted regression cases, controlled fault injection, and a bounded nightly workflow covering restart and teardown behavior. - Terminate reply observers, cancel telemetry collectors, bound dashboard projections, and release child observers, file descriptors, process records, and inode-verified Unix sockets on every terminal path. Verified with the compiler-policy contracts, 105 Myelin library tests, 32 swactor-process tests, telemetry cancellation contracts, randomized stateful restart cases, cargo check, and formatting checks.
2026-08-19 21:38:14 +00:00
// 9. Clean up poisoned and stopping actors. Cleanup changes the pool after
// phase 8 published its snapshot, so republish cardinality and mailbox
// state when actors were removed rather than leaving observability stale
// until unrelated work reaches this worker.
let cleaned_dead = Self::cleanup_dead_actors(
&mut self.pool,
&mut self.worker_ext,
&mut self.deferred_transfers,
&tc,
);
feat(myelin): enforce actor-owned control flow Architecture enforcement: - Install a repository-owned rustc wrapper for ordinary cargo check, build, and test commands. Resolve compiler item identities so renamed imports and helper wrappers cannot hide spawning, timing, blocking, polling, thread, or runtime-driving capabilities. - Define the execution-owner crates and reject dependencies from those substrates back into Myelin policy. Add compile-pass and compile-fail contracts for actor helpers, execution owners, test waits, forbidden capabilities, suppression attempts, and owner dependency inversions. Execution ownership: - Add engine-owned actor timers with cancellation and generation identity, then migrate lifecycle deadlines and protocol ticks off application tasks. Keep networking, process output, telemetry, and blocking provider calls in their approved I/O substrates. - Move process spawn, wait, signal, Unix listener, and output-following mechanics into swactor-process. Isolate Vast.ai blocking HTTP mechanics behind its adapter while actors retain retry, recovery, and provisioning decisions. Myelin control flow: - Rework manual control, worker lifecycle, provisioning, provider recovery, job deployment, distribution, edge orchestration, and shutdown as actor state transitions and typed effects. Preserve durable provider adoption and command outcomes across graceful and abrupt restarts. - Replace controller loops and timer-forwarding tasks with actor messages; leave substrate tasks as cancellable observation streams with no durable policy state. Properties and resource ownership: - Add deterministic engine and component properties, a stateful mock-VastAI lifecycle model, persisted regression cases, controlled fault injection, and a bounded nightly workflow covering restart and teardown behavior. - Terminate reply observers, cancel telemetry collectors, bound dashboard projections, and release child observers, file descriptors, process records, and inode-verified Unix sockets on every terminal path. Verified with the compiler-policy contracts, 105 Myelin library tests, 32 swactor-process tests, telemetry cancellation contracts, randomized stateful restart cases, cargo check, and formatting checks.
2026-08-19 21:38:14 +00:00
if cleaned_dead {
did_work = true;
self.stats
.num_actors
.store(self.pool.len(), Ordering::Relaxed);
self.stats
.total_mailbox_depth
.store(self.pool.total_mailbox_depth(), Ordering::Relaxed);
if let Some(hook) = tc.stats_hook {
self.pool.mailbox_depths_into(&mut self.snapshot_buf);
hook.on_tick(wid.index(), &self.snapshot_buf);
}
}
self.has_backlog = did_work;
did_work
}
fn drain_spawns(
pool: &mut ActorPool,
spawn_rx: &Receiver<SpawnRequest>,
tc: &TickContext,
budget: usize,
) -> bool {
let mut did_work = false;
let mut count = 0usize;
#[cfg(feature = "tracing")]
let mut spawn_count: usize = 0;
while let Some(mut req) = spawn_rx.try_recv() {
if let Some(ext) = tc.extension {
req.env = ext.on_spawn(
req.addr,
req.parent,
req.env,
tc.created_at.elapsed().as_millis() as u64,
);
}
pool.insert(req);
#[cfg(feature = "tracing")]
{
spawn_count += 1;
}
did_work = true;
count += 1;
if budget != 0 && count >= budget {
break;
}
}
#[cfg(feature = "tracing")]
if spawn_count > 0 {
tracing::debug!(
worker_id = tc.worker_id().index(),
count = spawn_count,
"worker.spawns_drained"
);
}
did_work
}
fn deliver_or_defer_transfer(
pool: &mut ActorPool,
tc: &TickContext,
deferred: &mut VecDeque<Envelope>,
envelope: Envelope,
) -> bool {
let dest = envelope.dest();
if pool.contains(&dest) {
pool.deliver(&dest, envelope.into_payload());
true
} else if tc.worker_of(&dest) == Some(tc.worker_id()) {
deferred.push_back(envelope);
false
} else {
true
}
}
fn drain_transfers(
pool: &mut ActorPool,
transfer_rx: &Receiver<Envelope>,
deferred: &mut VecDeque<Envelope>,
tc: &TickContext,
budget: usize,
) -> bool {
let mut did_work = false;
let mut count = 0usize;
while budget == 0 || count < budget {
let Some(envelope) = deferred.pop_front() else {
break;
};
if Self::deliver_or_defer_transfer(pool, tc, deferred, envelope) {
did_work = true;
}
count += 1;
}
while budget == 0 || count < budget {
let Some(envelope) = transfer_rx.try_recv() else {
break;
};
did_work = true;
Self::deliver_or_defer_transfer(pool, tc, deferred, envelope);
count += 1;
}
did_work
}
fn admin_target(cmd: &AdminCommand) -> Option<ActorAddress> {
match cmd {
AdminCommand::ListActors { .. } => None,
AdminCommand::InspectActor { actor, .. }
| AdminCommand::GetActorState { actor, .. }
| AdminCommand::ReplaceActorState { actor, .. }
| AdminCommand::StopActor { actor, .. }
| AdminCommand::SuspendActor { actor, .. }
| AdminCommand::ResumeActor { actor, .. } => Some(*actor),
}
}
fn should_defer_admin_command(pool: &ActorPool, tc: &TickContext, cmd: &AdminCommand) -> bool {
Self::admin_target(cmd).is_some_and(|actor| {
!pool.contains(&actor) && tc.worker_of(&actor) == Some(tc.worker_id())
})
}
fn apply_or_defer_admin_command(
pool: &mut ActorPool,
tc: &TickContext,
deferred: &mut VecDeque<AdminCommand>,
cmd: AdminCommand,
) -> bool {
if Self::should_defer_admin_command(pool, tc, &cmd) {
deferred.push_back(cmd);
false
} else {
Self::apply_admin_command(pool, tc, cmd);
true
}
}
fn drain_admin(
pool: &mut ActorPool,
admin_rx: &Receiver<AdminCommand>,
deferred: &mut VecDeque<AdminCommand>,
tc: &TickContext,
budget: usize,
) -> bool {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
let mut did_work = false;
let mut count = 0usize;
while budget == 0 || count < budget {
let Some(cmd) = deferred.pop_front() else {
break;
};
if Self::apply_or_defer_admin_command(pool, tc, deferred, cmd) {
did_work = true;
}
count += 1;
}
while budget == 0 || count < budget {
let Some(cmd) = admin_rx.try_recv() else {
break;
};
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
did_work = true;
Self::apply_or_defer_admin_command(pool, tc, deferred, cmd);
count += 1;
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
did_work
}
fn send_admin_reply<T: crate::actor::Message>(
tc: &TickContext,
reply_to: ActorAddress,
result: AdminResult<T>,
) {
let _ = tc.inbox_registry.try_deliver(reply_to, Box::new(result));
}
fn apply_admin_command(pool: &mut ActorPool, tc: &TickContext, cmd: AdminCommand) {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
match cmd {
AdminCommand::ListActors { acc } => {
let mut local = Vec::new();
pool.actor_summaries_into(&mut local, tc.worker_id);
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
{
let mut summaries = acc.summaries.lock();
summaries.extend(local);
}
if acc.remaining.fetch_sub(1, Ordering::AcqRel) == 1 {
let actors = {
let mut summaries = acc.summaries.lock();
std::mem::take(&mut *summaries)
};
Self::send_admin_reply(tc, acc.reply_to, Ok(ListActorsResponse { actors }));
}
}
AdminCommand::InspectActor { actor, reply_to } => {
let result = pool
.actor_summary(actor, tc.worker_id)
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
.map(|summary| InspectActorResponse { summary });
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::GetActorState {
actor,
reply_to,
get,
not_found,
} => {
let boxed = match pool.get_actor_erased(actor) {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
Some(erased) => get(actor, erased, erased.metadata()),
None => not_found(actor),
};
let _ = tc.inbox_registry.try_deliver(reply_to, boxed);
}
AdminCommand::ReplaceActorState {
actor,
reply_to,
replace,
} => {
let result = match pool.get_actor_erased_mut(actor) {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
Some(erased) => {
let metadata = erased.metadata();
replace(erased, metadata)
}
None => Err(AdminError::ActorNotFound { actor }),
};
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::StopActor { actor, reply_to } => {
let result = pool.stop_actor_admin(actor, tc.worker_stats);
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::SuspendActor { actor, reply_to } => {
let result = pool.suspend_actor_admin(actor);
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
Self::send_admin_reply(tc, reply_to, result);
}
AdminCommand::ResumeActor { actor, reply_to } => {
let result = pool.resume_actor_admin(actor);
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
Self::send_admin_reply(tc, reply_to, result);
}
}
}
/// Phase 9: clean up dead actors, deliver death notifications, GC extension state.
fn cleanup_dead_actors(
pool: &mut ActorPool,
worker_ext: &mut Option<Box<dyn WorkerExtension>>,
deferred_transfers: &mut VecDeque<Envelope>,
tc: &TickContext,
) -> bool {
let cleanup_pending: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
RefCell::new(Vec::new());
let cleanup_stops: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let cleanup_stop_withs: RefCell<Vec<(ActorAddress, ExitValue)>> = RefCell::new(Vec::new());
let cleanup_suspends: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let cleanup_requests: RefCell<Vec<Box<dyn Any + Send>>> = RefCell::new(Vec::new());
let dead = {
let cleanup_ctx = WorkerContext {
tc,
pending_local: &cleanup_pending,
stop_requests: &cleanup_stops,
stop_with_values: &cleanup_stop_withs,
suspend_requests: &cleanup_suspends,
worker_requests: &cleanup_requests,
stats: tc.worker_stats,
};
pool.cleanup_dead(&cleanup_ctx)
};
let had_dead = !dead.is_empty();
if had_dead {
for (addr, _, _) in &dead {
tc.address_map.remove(addr);
}
if let Some(ext) = tc.extension {
let notifications = ext.on_actor_death(&dead);
let dead_addrs: Vec<_> = dead.iter().map(|(a, _, _)| *a).collect();
ext.cleanup_dead(&dead_addrs);
for (dest, msg) in notifications {
route_runtime_message(pool, tc, dest, msg);
}
}
tc.worker_stats
.num_actors
.store(pool.len(), Ordering::Relaxed);
}
// Deliver any messages sent during on_stop callbacks.
for (addr, msg) in cleanup_pending.into_inner() {
Self::deliver_or_defer_transfer(pool, tc, deferred_transfers, Envelope::new(addr, msg));
}
// GC per-worker extension state for dead actors
if let Some(ext) = worker_ext {
let dead_addrs: Vec<ActorAddress> = dead.iter().map(|(a, _, _)| *a).collect();
ext.gc_dead(&dead_addrs);
}
had_dead
}
}
/// The `ContextInner` impl for in-worker sends.
///
/// Same-worker sends are staged in `pending_local` (eligible next pass).
/// Cross-worker sends move an `Envelope` into the target worker's transfer
/// queue. Non-actor addresses route to the inbox registry / transport seam.
struct WorkerContext<'a> {
tc: &'a TickContext<'a>,
pending_local: &'a RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>>,
stop_requests: &'a RefCell<Vec<ActorAddress>>,
stop_with_values: &'a RefCell<Vec<(ActorAddress, ExitValue)>>,
suspend_requests: &'a RefCell<Vec<ActorAddress>>,
worker_requests: &'a RefCell<Vec<Box<dyn Any + Send>>>,
stats: &'a WorkerStats,
}
impl ContextInner for WorkerContext<'_> {
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error> {
match self.tc.worker_of(&addr) {
Some(w) if w == self.tc.worker_id() => {
self.stats.local_sends.fetch_add(1, Ordering::Relaxed);
self.pending_local.borrow_mut().push((addr, msg));
Ok(())
}
Some(w) => {
// Cross-worker: move the payload through shared memory.
self.stats.local_sends.fetch_add(1, Ordering::Relaxed);
self.tc.transfer_tx(w).send(Envelope::new(addr, msg));
Ok(())
}
None => {
self.stats.inbox_sends.fetch_add(1, Ordering::Relaxed);
self.tc.route_nonlocal(addr, msg)
}
}
}
fn spawn_any(&self, request: SpawnRequest) {
// ctx.spawn pins the child to the current worker.
self.tc
.address_map
.insert(request.addr, self.tc.worker_id());
self.tc.spawn_tx(self.tc.worker_id()).send(request);
}
fn request_stop(&self, addr: ActorAddress) {
self.stop_requests.borrow_mut().push(addr);
}
fn request_stop_with(&self, addr: ActorAddress, value: ExitValue) {
self.stop_with_values.borrow_mut().push((addr, value));
}
fn request_suspend(&self, addr: ActorAddress) {
self.suspend_requests.borrow_mut().push(addr);
}
fn request_resume(&self, addr: ActorAddress) {
self.pending_local
.borrow_mut()
.push((addr, Box::new(ResumeSignal)));
}
fn post_worker_request(&self, request: Box<dyn Any + Send>) {
self.worker_requests.borrow_mut().push(request);
}
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
self.tc.extension
}
fn process_output_observer(
&self,
) -> Option<std::sync::Arc<dyn crate::process_observer::ProcessOutputObserver>> {
self.tc.process_output_observer.cloned()
}
fn system_info(&self) -> SystemInfo {
SystemInfo {
worker_id: self.tc.worker_id().index(),
num_workers: self.tc.num_workers,
total_actors: self.tc.address_map.len(),
uptime_ms: self.tc.created_at.elapsed().as_millis() as u64,
}
}
}
struct ActorSlot {
mailbox: VecDeque<Box<dyn Any + Send>>,
actor: Box<dyn AnyActor>,
poisoned: bool,
/// Graceful stop requested (via StopSignal).
stopping: bool,
/// Whether on_start has been called for this actor.
started: bool,
/// Actor is suspended — messages queue but are not processed.
suspended: bool,
last_msg_type: Option<&'static str>,
messages_processed: u64,
/// Per-message-type counters (bounded to 32 entries).
msg_type_counts: HashMap<&'static str, u64>,
/// Address of the actor that spawned this one, or `None` for externally-spawned actors.
parent_addr: Option<ActorAddress>,
/// Inherited environment from parent (or empty for runtime-spawned actors).
env: Environment,
/// Typed exit value set by `ctx.stop_with()`.
exit_value: Option<ExitValue>,
}
/// Per-worker actor storage. Owns per-actor mailboxes.
pub(crate) struct ActorPool {
actors: AddrMap<ActorSlot>,
}
impl ActorPool {
pub fn new() -> Self {
Self {
actors: HashMap::with_hasher(AddrBuildHasher),
}
}
pub fn insert(&mut self, req: SpawnRequest) {
self.actors.insert(
req.addr,
ActorSlot {
mailbox: VecDeque::with_capacity(16),
actor: req.actor,
poisoned: false,
stopping: false,
started: false,
suspended: false,
last_msg_type: None,
messages_processed: 0,
msg_type_counts: HashMap::new(),
parent_addr: req.parent,
env: req.env,
exit_value: None,
},
);
}
/// Deliver a type-erased message to the actor at `addr`.
/// Returns `true` if the actor exists (message enqueued; type check deferred to tick).
pub fn deliver(&mut self, addr: &ActorAddress, msg: Box<dyn Any + Send>) -> bool {
if let Some(slot) = self.actors.get_mut(addr) {
// Intercept control signals for suspended actors: they skip tick_all
// so we must handle resume/stop at delivery time.
if slot.suspended {
if msg.is::<ResumeSignal>() {
slot.suspended = false;
return true;
}
if msg.is::<StopSignal>() {
slot.stopping = true;
slot.mailbox.clear();
return true;
}
if msg.is::<StopWithSignal>() {
if let Ok(sig) = msg.downcast::<StopWithSignal>() {
slot.exit_value = Some(sig.0);
}
slot.stopping = true;
slot.mailbox.clear();
return true;
}
}
slot.mailbox.push_back(msg);
true
} else {
false
}
}
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
fn get_actor_erased(&self, addr: ActorAddress) -> Option<&dyn AnyActor> {
self.actors.get(&addr).map(|slot| slot.actor.as_ref())
}
fn get_actor_erased_mut(
&mut self,
addr: ActorAddress,
) -> Option<&mut (dyn AnyActor + 'static)> {
self.actors.get_mut(&addr).map(|slot| slot.actor.as_mut())
}
fn actor_summary_from_slot(
address: ActorAddress,
slot: &ActorSlot,
worker: WorkerId,
) -> ActorSummary {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
let metadata = slot.actor.metadata();
ActorSummary {
address,
actor_type: metadata.actor_type_name,
message_type: metadata.message_type_name,
worker_id: worker.index(),
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
parent: slot.parent_addr,
mailbox_depth: slot.mailbox.len(),
status: ActorStatus {
started: slot.started,
suspended: slot.suspended,
stopping: slot.stopping,
poisoned: slot.poisoned,
},
last_message_type: slot.last_msg_type,
messages_handled: slot.messages_processed,
}
}
fn actor_summary(&self, addr: ActorAddress, worker: WorkerId) -> AdminResult<ActorSummary> {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
self.actors
.get(&addr)
.map(|slot| Self::actor_summary_from_slot(addr, slot, worker))
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
.ok_or(AdminError::ActorNotFound { actor: addr })
}
fn actor_summaries_into(&self, out: &mut Vec<ActorSummary>, worker: WorkerId) {
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
out.clear();
out.extend(
self.actors
.iter()
.map(|(&addr, slot)| Self::actor_summary_from_slot(addr, slot, worker)),
feat: actor direct control via runtime admin Add a RuntimeAdmin control plane for out-of-band inspection and mutation of live actors, bypassing their normal message handlers. - src/admin.rs: add RuntimeAdmin + Admin<T> reply handle, the AdminCommand enum (ListActors, InspectActor, GetActorState, ReplaceActorState, StopActor, SuspendActor, ResumeActor), typed result types (ActorSummary/ActorStatus/ActorStateSnapshot/ListActorsResponse), and AdminError (ActorNotFound/AddressMismatch/TypeMismatch/Timeout) - src/actor.rs: expose type-erased accessors on AnyActor — metadata() returning ActorTypeMetadata plus as_any()/as_any_mut() for downcasting — and Actor::inner()/replace_inner() so admin can snapshot or swap concrete actor state - src/runtime.rs: wire a per-worker admin channel (admin_txs), expose Runtime::admin(), and implement the RuntimeAdmin ops that resolve an address to its owning worker, send the command, notify that worker, and return an Admin<T> handle; Admin::recv_ticking drives ticks and collects the reply - src/worker.rs: add an admin_rx receiver and a drain_admin tick phase run before actor handlers, plus ActorPool helpers (actor_summary, get_actor_erased[_mut], suspend/resume/stop_actor_admin) that apply commands on the worker thread; fold admin_rx into the fast-idle has_work/tick_once checks - tests/runtime_admin.rs: add 608 lines of end-to-end coverage for list/inspect/get/replace/stop/suspend/resume and the type-mismatch and not-found error paths Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-08 15:23:03 +00:00
);
}
fn suspend_actor_admin(&mut self, addr: ActorAddress) -> AdminResult<OperationResult> {
match self.actors.get_mut(&addr) {
Some(slot) => {
slot.suspended = true;
Ok(OperationResult { applied: true })
}
None => Err(AdminError::ActorNotFound { actor: addr }),
}
}
fn resume_actor_admin(&mut self, addr: ActorAddress) -> AdminResult<OperationResult> {
match self.actors.get_mut(&addr) {
Some(slot) => {
slot.suspended = false;
Ok(OperationResult { applied: true })
}
None => Err(AdminError::ActorNotFound { actor: addr }),
}
}
fn stop_actor_admin(
&mut self,
addr: ActorAddress,
stats: &WorkerStats,
) -> AdminResult<OperationResult> {
match self.actors.get_mut(&addr) {
Some(slot) => {
if !slot.stopping {
stats.stops.fetch_add(1, Ordering::Relaxed);
}
slot.stopping = true;
slot.mailbox.clear();
Ok(OperationResult { applied: true })
}
None => Err(AdminError::ActorNotFound { actor: addr }),
}
}
/// Tick all actors in the pool. Returns the number of messages processed.
///
/// Each actor processes up to `budget` messages per tick (0 = unlimited).
/// This prevents a single hot actor from starving others on the same worker.
fn tick_all(&mut self, wctx: &WorkerContext<'_>, stats: &WorkerStats, budget: usize) -> usize {
let mut count = 0;
for (&addr, slot) in self.actors.iter_mut() {
if should_skip_actor(slot.poisoned, slot.stopping, slot.suspended) {
// Discard all messages for poisoned/stopping actors (not suspended — those queue)
if slot.poisoned || slot.stopping {
slot.mailbox.clear();
}
continue;
}
debug_assert!(
!slot.poisoned && !slot.stopping && !slot.suspended,
"G4: non-processable actor reached processing"
);
#[cfg(feature = "tracing")]
let _actor_span = tracing::trace_span!("actor.tick", actor_addr = %addr).entered();
// Snapshot self-stats before creating Ctx
let snap_processed = slot.messages_processed;
let snap_depth = slot.mailbox.len();
let mut snap_type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
snap_type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let ctx = Ctx::new(
wctx,
addr,
slot.parent_addr,
slot.env.clone(),
snap_processed,
snap_depth,
snap_type_counts,
);
// Call on_start once, before first message
if !slot.started {
let start_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.on_start(&ctx);
}));
slot.started = true;
if start_result.is_err() {
stats.panics.fetch_add(1, Ordering::Relaxed);
eprintln!("swactor: actor {addr} panicked in on_start — poisoned");
#[cfg(feature = "tracing")]
tracing::error!(actor_addr = %addr, "actor.on_start_panicked");
slot.poisoned = true;
slot.mailbox.clear();
continue;
}
// Check if on_start requested stop or stop_with
{
let stops = wctx.stop_requests.borrow();
if !stops.is_empty() && stops.contains(&addr) {
drop(stops);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
// Check for stop_with value
let mut sws = wctx.stop_with_values.borrow_mut();
if let Some(pos) = sws.iter().position(|(a, _)| *a == addr) {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
}
continue;
}
}
// Check if on_start requested stop_with (without plain stop)
{
let mut sws = wctx.stop_with_values.borrow_mut();
if let Some(pos) = sws.iter().position(|(a, _)| *a == addr) {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
continue;
}
}
// Check if on_start requested suspend
{
let suspends = wctx.suspend_requests.borrow();
if !suspends.is_empty() && suspends.contains(&addr) {
drop(suspends);
slot.suspended = true;
continue;
}
}
}
let mut actor_count = 0usize;
while let Some(msg) = slot.mailbox.pop_front() {
// Intercept StopSignal (from external runtime.stop_actor)
if msg.is::<StopSignal>() {
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
#[cfg(feature = "tracing")]
tracing::info!(actor_addr = %addr, "actor.stop_requested");
break;
}
// Intercept StopWithSignal (from external runtime)
if msg.is::<StopWithSignal>() {
if let Ok(sig) = msg.downcast::<StopWithSignal>() {
slot.exit_value = Some(sig.0);
}
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
break;
}
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.handle_any(&ctx, msg)
}));
match result {
Ok(None) => {
stats.type_mismatches.fetch_add(1, Ordering::Relaxed);
}
Err(_) => {
stats.panics.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
eprintln!(
"swactor: actor {addr} panicked — poisoned, future messages will be discarded"
);
#[cfg(feature = "tracing")]
tracing::error!(actor_addr = %addr, "actor.panicked");
slot.poisoned = true;
slot.mailbox.clear();
break;
}
Ok(Some(type_name)) => {
slot.last_msg_type = Some(type_name);
slot.messages_processed += 1;
// Track per-type counts (bounded to 32 distinct types)
if slot.msg_type_counts.len() < 32
|| slot.msg_type_counts.contains_key(type_name)
{
*slot.msg_type_counts.entry(type_name).or_insert(0) += 1;
}
}
}
count += 1;
actor_count += 1;
// Check if handler requested self-stop or stop_with
{
let stops = wctx.stop_requests.borrow();
let has_stop = !stops.is_empty() && stops.contains(&addr);
drop(stops);
let mut sws = wctx.stop_with_values.borrow_mut();
let sw_pos = sws.iter().position(|(a, _)| *a == addr);
if has_stop || sw_pos.is_some() {
if let Some(pos) = sw_pos {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
}
drop(sws);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
break;
}
drop(sws);
}
// Check if handler requested suspend
{
let suspends = wctx.suspend_requests.borrow();
if !suspends.is_empty() && suspends.contains(&addr) {
drop(suspends);
slot.suspended = true;
break; // stop processing this actor's messages this tick
}
}
if budget > 0 && actor_count >= budget {
break;
}
}
}
count
}
pub fn len(&self) -> usize {
self.actors.len()
}
pub fn contains(&self, addr: &ActorAddress) -> bool {
self.actors.contains_key(addr)
}
pub fn total_mailbox_depth(&self) -> usize {
self.actors.values().map(|slot| slot.mailbox.len()).sum()
}
/// Remove poisoned and stopping actors, returning their addresses, stop reasons,
/// and optional exit values.
/// Called after tick_all so the caller can clean up the address map.
///
/// For stopping actors: calls `on_stop()` before removal (wrapped in catch_unwind).
/// For poisoned actors: `on_stop()` is NOT called (state may be corrupt).
pub fn cleanup_dead(
&mut self,
inner: &dyn ContextInner,
) -> Vec<(ActorAddress, StopReason, Option<ExitValue>)> {
let dead_addrs: Vec<ActorAddress> = self
.actors
.iter()
.filter(|(_, slot)| slot.poisoned || slot.stopping)
.map(|(&addr, _)| addr)
.collect();
let mut dead = Vec::with_capacity(dead_addrs.len());
for addr in dead_addrs {
if let Some(mut slot) = self.actors.remove(&addr) {
let reason = determine_stop_reason(slot.poisoned, slot.exit_value.is_some());
// Call on_stop for gracefully stopping actors only
debug_assert!(
slot.poisoned || slot.stopping,
"G4: non-dead actor reached cleanup_dead"
);
if is_on_stop_eligible(slot.stopping, slot.poisoned) {
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let ctx = Ctx::new(
inner,
addr,
slot.parent_addr,
slot.env.clone(),
slot.messages_processed,
slot.mailbox.len(),
type_counts,
);
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.on_stop(&ctx);
}));
}
dead.push((addr, reason, slot.exit_value.take()));
// slot is dropped here — actor resources freed
}
}
dead
}
/// Fill `out` with per-actor snapshots, reusing the existing allocation.
pub fn mailbox_depths_into(&self, out: &mut Vec<ActorSnapshot>) {
out.clear();
out.extend(self.actors.iter().map(|(&addr, slot)| {
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let metadata = slot.actor.metadata();
ActorSnapshot {
address: addr,
mailbox_depth: slot.mailbox.len(),
last_msg_type: slot.last_msg_type,
actor_type: Some(metadata.actor_type_name),
message_type: Some(metadata.message_type_name),
messages_processed: slot.messages_processed,
poisoned: slot.poisoned,
message_type_counts: type_counts,
}
}));
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::actor::StopReason;
#[test]
fn lifecycle_decision_helpers_cover_all_inputs() {
let mut skip_combinations = 0;
for poisoned in [false, true] {
for stopping in [false, true] {
for suspended in [false, true] {
assert_eq!(
should_skip_actor(poisoned, stopping, suspended),
poisoned || stopping || suspended
);
skip_combinations += 1;
}
}
}
assert_eq!(skip_combinations, 8);
let mut on_stop_combinations = 0;
for stopping in [false, true] {
for poisoned in [false, true] {
assert_eq!(
is_on_stop_eligible(stopping, poisoned),
stopping && !poisoned
);
on_stop_combinations += 1;
}
}
assert_eq!(on_stop_combinations, 4);
let mut reason_combinations = 0;
for poisoned in [false, true] {
for has_exit_value in [false, true] {
let expected = if poisoned {
StopReason::Panicked
} else if has_exit_value {
StopReason::Completed
} else {
StopReason::Normal
};
assert_eq!(determine_stop_reason(poisoned, has_exit_value), expected);
reason_combinations += 1;
}
}
assert_eq!(reason_combinations, 4);
}
#[test]
fn bounded_lifecycle_model_preserves_callback_invariants() {
#[derive(Clone, Copy)]
enum Step {
Tick,
RequestStop,
Poison,
Suspend,
Resume,
Cleanup,
}
#[derive(Clone, Copy, Default)]
struct ActorProbe {
started: bool,
stopping: bool,
poisoned: bool,
suspended: bool,
removed: bool,
on_start_count: u8,
handle_count: u8,
on_stop_count: u8,
handled_after_on_stop: bool,
}
impl ActorProbe {
fn apply(&mut self, step: Step) {
if self.removed {
return;
}
match step {
Step::Tick => {
let eligible =
!should_skip_actor(self.poisoned, self.stopping, self.suspended);
if !self.started && eligible {
self.started = true;
self.on_start_count += 1;
}
if self.started && eligible {
if self.on_stop_count > 0 {
self.handled_after_on_stop = true;
}
self.handle_count += 1;
}
}
Step::RequestStop => {
self.stopping = true;
}
Step::Poison => {
self.poisoned = true;
}
Step::Suspend => {
self.suspended = true;
}
Step::Resume => {
self.suspended = false;
}
Step::Cleanup => {
if self.stopping || self.poisoned {
if is_on_stop_eligible(self.stopping, self.poisoned) {
self.on_stop_count += 1;
}
self.removed = true;
}
}
}
}
fn assert_invariants(self) {
assert!(self.on_start_count <= 1, "on_start fired more than once");
assert!(self.on_stop_count <= 1, "on_stop fired more than once");
assert!(
!self.handled_after_on_stop,
"handle fired after on_stop cleanup"
);
if self.handle_count > 0 {
assert!(self.started, "handle fired before on_start");
}
if self.on_stop_count > 0 {
assert!(self.removed, "on_stop fired without cleanup");
assert!(!self.poisoned, "poisoned actor ran on_stop");
}
}
}
fn walk(depth: usize, state: ActorProbe, checked: &mut usize) {
state.assert_invariants();
*checked += 1;
if depth == 0 {
return;
}
for step in [
Step::Tick,
Step::RequestStop,
Step::Poison,
Step::Suspend,
Step::Resume,
Step::Cleanup,
] {
let mut next = state;
next.apply(step);
walk(depth - 1, next, checked);
}
}
let mut checked = 0;
walk(6, ActorProbe::default(), &mut checked);
assert_eq!(checked, 55_987);
}
}