feat: actor view panels for the dashboard

Add read-only actor overview and per-actor dossier pages to the dashboard, fed by enriched per-actor runtime snapshots.

- `dashboard/swactor/actor_view`: new `ActorPanelView`, a tolerant frame consumer over `runtime.actors`/`runtime.stats` that folds per-actor snapshots and serves `/view/swactor/actor-overview` (roster) and `/view/swactor/actor-dossier` (per-actor detail), each backed by an embedded HTML template (`actor_overview.html`, `actor_dossier.html`)
- `dashboard`: register both views in `DashboardHandle` and export `actor_overview_view()`/`actor_dossier_view()` from the swactor module
- `swactor` core: enrich `ActorSnapshot` with `actor_type` and `message_type` (populated from `slot.actor.metadata()` in `ActorPool`) and add `ActorAddress::to_full_hex()` for untruncated display
- `myelin/orchestration`: publish actor stats to the dashboard via a `runtime.actors` channel producer (`stats_hook_on`) threaded through the distribution stack, and carry the orchestrator actor address into readiness signaling
- workspace `Cargo.toml`: add `default-members` for native iteration and a centralized `[workspace.dependencies] tokio` so members share one feature set; `dashboard/Cargo.toml` switches to `tokio.workspace = true`

Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-08-03 14:18:24 +04:00
parent ef9e1c98a3
commit 2bf11e0fdb
35 changed files with 1490 additions and 140 deletions

View file

@ -15,7 +15,27 @@ members = [
"xtask", "xtask",
"tools/vastai", "tools/vastai",
] ]
default-members = [
".",
"crates/process",
"crates/provisioning",
"crates/transport",
"crates/distribution",
"crates/iroh-driver",
"crates/datastream",
"crates/data-plane",
"crates/dashboard",
"apps/myelin",
"tools/vastai",
]
# The language bindings (python, wasm) and xtask are built on demand
# (-p / --workspace / cargo-xtask), not during normal native iteration.
exclude = ["crates/bindings/wasm-crypto", "examples"] exclude = ["crates/bindings/wasm-crypto", "examples"]
[workspace.dependencies]
# Centralized so every member resolves tokio with the SAME feature set.
# Without this, members declared tokio with different features, so building
# one member vs another (or `run` vs `test`) recompiled tokio each time.
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
[package] [package]
name = "swactor" name = "swactor"

View file

@ -22,7 +22,7 @@ swactor-process = { path = "../../crates/process" }
distribution = { path = "../../crates/distribution" } distribution = { path = "../../crates/distribution" }
iroh-driver = { path = "../../crates/iroh-driver" } iroh-driver = { path = "../../crates/iroh-driver" }
iroh = "0.98" iroh = "0.98"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] } tokio.workspace = true
swactor-vastai = { path = "../../tools/vastai" } swactor-vastai = { path = "../../tools/vastai" }
parking_lot = "0.12" parking_lot = "0.12"
blake3 = "1" blake3 = "1"

View file

@ -56,7 +56,7 @@ OPTIONS:
--relay-mode <mode> Relay mode: default or disabled --relay-mode <mode> Relay mode: default or disabled
--relay-url <url> Custom relay URL passed to myelin-orchestrator --relay-url <url> Custom relay URL passed to myelin-orchestrator
--endpoint-addr-mask <mask> Endpoint address mask: full or relay-only --endpoint-addr-mask <mask> Endpoint address mask: full or relay-only
--cached-model[=<path>] Use discovered or explicit cached GGUF model --cached-model[=<path>] Use discovered or explicit cached GGUF model (default for --process)
--dump-logs[=<path>] Write datastream frame log --dump-logs[=<path>] Write datastream frame log
--run-id <id> Override run id --run-id <id> Override run id
--skip-rebuild Reuse existing Cargo artifacts --skip-rebuild Reuse existing Cargo artifacts
@ -793,9 +793,15 @@ impl Config {
let gpu_run = args.gpu || env_flag(MYELIN_CHAT_GPU_RUN_ENV, false); let gpu_run = args.gpu || env_flag(MYELIN_CHAT_GPU_RUN_ENV, false);
let endpoint_addr_mask = Self::endpoint_addr_mask(&args, &toml)?; let endpoint_addr_mask = Self::endpoint_addr_mask(&args, &toml)?;
let (relay_mode, relay_url) = Self::relay_settings(&args, &toml, endpoint_addr_mask)?; let (relay_mode, relay_url) = Self::relay_settings(&args, &toml, endpoint_addr_mask)?;
let cached_model = Self::cached_model_source(&args, gpu_run, &provider) let cached_model = match Self::cached_model_source(&args, &provider) {
.map(CachedModelConfig::from_source) None => None,
.transpose()?; Some(CachedModelSource::Path(path)) => Some(CachedModelConfig::from_path(path)?),
Some(CachedModelSource::Discover) => match CachedModelConfig::discover() {
Ok(config) => Some(config),
Err(error) if args.cached_model.is_some() => return Err(error),
Err(_) => None,
},
};
let model = Self::model_config(&provider, &toml, cached_model.as_ref())?; let model = Self::model_config(&provider, &toml, cached_model.as_ref())?;
let datastream_frame_log = Self::datastream_frame_log(&args, &toml); let datastream_frame_log = Self::datastream_frame_log(&args, &toml);
let vastai = if provider == provider_kind::vastai() { let vastai = if provider == provider_kind::vastai() {
@ -876,16 +882,15 @@ impl Config {
Ok((relay_mode, relay_url)) Ok((relay_mode, relay_url))
} }
fn cached_model_source( /// Resolve the cached-model source. The process provider defaults to
args: &ParsedArgs, /// best-effort discovery of `.model-cache/` so `cargo myelin-chat` runs a
gpu_run: bool, /// cached GGUF model without explicit flags. Explicit `--cached-model` is
provider: &ProviderKind, /// always honored (and stays strict); the default degrades gracefully to
) -> Option<CachedModelSource> { /// the normal download path when no cached model is present.
fn cached_model_source(args: &ParsedArgs, provider: &ProviderKind) -> Option<CachedModelSource> {
match &args.cached_model { match &args.cached_model {
Some(source) => Some(source.clone()), Some(source) => Some(source.clone()),
None if gpu_run && provider == &provider_kind::process() => { None if provider == &provider_kind::process() => Some(CachedModelSource::Discover),
Some(CachedModelSource::Discover)
}
None => None, None => None,
} }
} }
@ -2122,13 +2127,6 @@ struct CachedModelConfig {
} }
impl CachedModelConfig { impl CachedModelConfig {
fn from_source(source: CachedModelSource) -> Result<Self, String> {
match source {
CachedModelSource::Discover => Self::discover(),
CachedModelSource::Path(path) => Self::from_path(path),
}
}
fn from_path(path: PathBuf) -> Result<Self, String> { fn from_path(path: PathBuf) -> Result<Self, String> {
let metadata = fs::metadata(&path) let metadata = fs::metadata(&path)
.map_err(|e| format!("stat cached model {}: {e}", path.display()))?; .map_err(|e| format!("stat cached model {}: {e}", path.display()))?;
@ -2517,6 +2515,34 @@ tag = " alias "
); );
} }
#[test]
fn config_defaults_use_cached_model_for_process_when_present() {
let temp = TempDir::new("cached-model-default");
with_process_state(&[], Some(temp.path()), || {
let cache_dir = temp.path().join(REPO_MODEL_CACHE_DIR);
fs::create_dir_all(&cache_dir).expect("create model cache dir");
fs::write(
cache_dir.join(DEFAULT_PIPELINE_CACHED_MODEL_FILE),
Vec::<u8>::new(),
)
.expect("seed cached model file");
let defaults = Config::from_args(Vec::<String>::new()).expect("defaults resolve");
assert_eq!(defaults.provider, provider_kind::process());
let cached_model = defaults
.cached_model
.expect("process provider discovers a cached model by default");
assert!(
cached_model
.host_path
.ends_with(DEFAULT_PIPELINE_CACHED_MODEL_FILE),
"discovered the seeded cached model"
);
});
}
#[test] #[test]
fn config_max_tokens_drives_orchestrator_args_and_submit_prompt() { fn config_max_tokens_drives_orchestrator_args_and_submit_prompt() {
let temp = TempDir::new("config-max-tokens"); let temp = TempDir::new("config-max-tokens");

View file

@ -170,6 +170,19 @@ pub(crate) enum NodeAgentMsg {
WorkerCrashed { WorkerCrashed {
reason: Option<String>, reason: Option<String>,
}, },
StepFailed {
step_id: u64,
},
ObjectFailed {
edge_id: u64,
object_id: Option<u64>,
},
OutputFault {
edge_id: u64,
},
EdgeFault {
edge_id: u64,
},
StopRun { StopRun {
run_id: u64, run_id: u64,
}, },
@ -236,9 +249,6 @@ pub(crate) enum StageCommandWire {
layer_end_exclusive: u32, layer_end_exclusive: u32,
stage_shard_plan: Option<StageShardPlan>, stage_shard_plan: Option<StageShardPlan>,
}, },
RewireEdge {
edge_id: u64,
},
ExecuteStep { ExecuteStep {
step_id: u64, step_id: u64,
input_edge_id: u64, input_edge_id: u64,
@ -531,6 +541,27 @@ impl NodeAgentActor {
self.last_worker_crash = reason; self.last_worker_crash = reason;
self.core.observe(stage::StageEvent::WorkerCrashed) self.core.observe(stage::StageEvent::WorkerCrashed)
} }
NodeAgentMsg::StepFailed { step_id } => {
self.core.observe(stage::StageEvent::StepFailed {
step_id: stage::StepId(step_id),
})
}
NodeAgentMsg::ObjectFailed { edge_id, object_id } => {
self.core.observe(stage::StageEvent::ObjectFailed {
edge_id: stage::EdgeId(edge_id),
object_id: object_id.map(stage::ObjectId),
})
}
NodeAgentMsg::OutputFault { edge_id } => {
self.core.observe(stage::StageEvent::OutputFault {
edge_id: stage::EdgeId(edge_id),
})
}
NodeAgentMsg::EdgeFault { edge_id } => {
self.core.observe(stage::StageEvent::EdgeFault {
edge_id: stage::EdgeId(edge_id),
})
}
NodeAgentMsg::StopRun { run_id } => self.core.observe(stage::StageEvent::StopRun { NodeAgentMsg::StopRun { run_id } => self.core.observe(stage::StageEvent::StopRun {
run_id: stage::RunId(run_id), run_id: stage::RunId(run_id),
}), }),
@ -692,7 +723,6 @@ impl From<&stage::StageCommand> for StageCommandWire {
layer_end_exclusive: range.end_exclusive, layer_end_exclusive: range.end_exclusive,
stage_shard_plan: shard_plan.clone(), stage_shard_plan: shard_plan.clone(),
}, },
stage::StageCommand::RewireEdge { edge_id } => Self::RewireEdge { edge_id: edge_id.0 },
stage::StageCommand::ExecuteStep(step) => Self::ExecuteStep { stage::StageCommand::ExecuteStep(step) => Self::ExecuteStep {
step_id: step.step_id.0, step_id: step.step_id.0,
input_edge_id: step.input.edge_id.0, input_edge_id: step.input.edge_id.0,

View file

@ -1013,7 +1013,7 @@ impl WorkerEdgeRuntime {
); );
let step_started = Instant::now(); let step_started = Instant::now();
let mut pump = || pump_network(driver, stack); let mut pump = || pump_network(driver, stack);
let committed_bytes = worker.execute_step( let committed_bytes = match worker.execute_step(
u64::from(config.stage_index) + 1, u64::from(config.stage_index) + 1,
step_id, step_id,
object_id, object_id,
@ -1027,7 +1027,15 @@ impl WorkerEdgeRuntime {
config, config,
datastream, datastream,
&mut pump, &mut pump,
)?; ) {
Ok(bytes) => bytes,
Err(e) => {
let _ = stack
.runtime
.send_to(node_actor, NodeAgentMsg::StepFailed { step_id });
return Err(e);
}
};
let helper_execute_ms = duration_ms_u64(step_started.elapsed()); let helper_execute_ms = duration_ms_u64(step_started.elapsed());
let egress_read_started = Instant::now(); let egress_read_started = Instant::now();
let record = { let record = {
@ -1035,9 +1043,16 @@ impl WorkerEdgeRuntime {
let lease = arena let lease = arena
.lookup_lease(arena::RingId(output_ring_id)) .lookup_lease(arena::RingId(output_ring_id))
.ok_or_else(|| format!("outbound ring {output_ring_id} lease missing"))?; .ok_or_else(|| format!("outbound ring {output_ring_id} lease missing"))?;
arena arena.read_arena(lease.layout.data_offset, committed_bytes)
.read_arena(lease.layout.data_offset, committed_bytes) };
.map_err(|e| format!("read egress ring: {e}"))? let record = match record {
Ok(record) => record,
Err(e) => {
let _ = stack
.runtime
.send_to(node_actor, NodeAgentMsg::OutputFault { edge_id: outbound.edge_id });
return Err(format!("read egress ring: {e}"));
}
}; };
let egress_read_ms = duration_ms_u64(egress_read_started.elapsed()); let egress_read_ms = duration_ms_u64(egress_read_started.elapsed());
let record_bytes = record.len(); let record_bytes = record.len();
@ -1067,7 +1082,12 @@ impl WorkerEdgeRuntime {
.as_ref() .as_ref()
.ok_or_else(|| "outbound edge sender missing".to_owned())?; .ok_or_else(|| "outbound edge sender missing".to_owned())?;
let edge_send_started = Instant::now(); let edge_send_started = Instant::now();
sender.send(record)?; if let Err(e) = sender.send(record) {
let _ = stack
.runtime
.send_to(node_actor, NodeAgentMsg::OutputFault { edge_id: outbound.edge_id });
return Err(e);
}
let edge_send_ms = duration_ms_u64(edge_send_started.elapsed()); let edge_send_ms = duration_ms_u64(edge_send_started.elapsed());
node_stage( node_stage(
datastream, datastream,
@ -1131,8 +1151,17 @@ impl WorkerEdgeRuntime {
buffer.extend_from_slice(&bytes); buffer.extend_from_slice(&bytes);
let buffered_bytes = buffer.len(); let buffered_bytes = buffer.len();
let mut records = Vec::new(); let mut records = Vec::new();
while let Some(record) = take_complete_ingress_record(buffer, inbound.object_spec)? { loop {
records.push(record); match take_complete_ingress_record(buffer, inbound.object_spec) {
Ok(Some(record)) => records.push(record),
Ok(None) => break,
Err(e) => {
let _ = stack
.runtime
.send_to(node_actor, NodeAgentMsg::ObjectFailed { edge_id, object_id: None });
return Err(e);
}
}
} }
(records, buffered_bytes) (records, buffered_bytes)
}; };
@ -1171,14 +1200,22 @@ impl WorkerEdgeRuntime {
}), }),
); );
let object_load_started = Instant::now(); let object_load_started = Instant::now();
let loaded = worker.ring_readable( let loaded = match worker.ring_readable(
ring_id, ring_id,
edge_id, edge_id,
inbound.object_spec, inbound.object_spec,
config, config,
datastream, datastream,
&mut || {}, &mut || {},
)?; ) {
Ok(loaded) => loaded,
Err(e) => {
let _ = stack
.runtime
.send_to(node_actor, NodeAgentMsg::ObjectFailed { edge_id, object_id: Some(record.object_id) });
return Err(e);
}
};
let object_load_ms = duration_ms_u64(object_load_started.elapsed()); let object_load_ms = duration_ms_u64(object_load_started.elapsed());
let key = ObjectKey { let key = ObjectKey {
edge_id, edge_id,
@ -1510,11 +1547,9 @@ impl WorkerEdgeRuntime {
.runtime .runtime
.send_to( .send_to(
node_actor, node_actor,
NodeAgentMsg::WorkerCrashed { NodeAgentMsg::EdgeFault { edge_id: edge_id.0 },
reason: Some(format!("edge {} faulted: {reason:?}", edge_id.0)),
},
) )
.map_err(|e| format!("mark worker crashed after edge fault: {e}"))?; .map_err(|e| format!("report edge fault: {e}"))?;
return Err(format!("edge {} faulted: {reason:?}", edge_id.0)); return Err(format!("edge {} faulted: {reason:?}", edge_id.0));
} }
edge::EdgeLifecycleEvent::EdgeStopped { .. } => {} edge::EdgeLifecycleEvent::EdgeStopped { .. } => {}
@ -1691,6 +1726,8 @@ fn run() -> Result<(), String> {
)?; )?;
} }
let mut datastream = NodeDatastream::new(&config);
let worker_stats_hook = datastream.producer.stats_hook();
let stack = DistributionRuntimeStack::new_with_codecs( let stack = DistributionRuntimeStack::new_with_codecs(
driver.node_id(), driver.node_id(),
DistributedNodeConfig::default(), DistributedNodeConfig::default(),
@ -1698,6 +1735,7 @@ fn run() -> Result<(), String> {
register_myelin_actor_codecs(registry); register_myelin_actor_codecs(registry);
datastream::wire::register_datastream_codec(registry); datastream::wire::register_datastream_codec(registry);
}, },
Some(worker_stats_hook),
); );
boot( boot(
"distribution_stack", "distribution_stack",
@ -1750,7 +1788,6 @@ fn run() -> Result<(), String> {
}; };
let arena_fd = arena_manager.lock().arena_fd(); let arena_fd = arena_manager.lock().arena_fd();
let mut datastream = NodeDatastream::new(&config);
let node_boot = |ds: &mut NodeDatastream, phase: &str, status: &str, detail: Value| { let node_boot = |ds: &mut NodeDatastream, phase: &str, status: &str, detail: Value| {
emit_node_event(ds, &config, NODE_BOOTSTRAP_CHANNEL, phase, status, detail) emit_node_event(ds, &config, NODE_BOOTSTRAP_CHANNEL, phase, status, detail)
}; };
@ -3560,15 +3597,6 @@ fn handle_stage_command(
); );
Ok(()) Ok(())
} }
StageCommandWire::RewireEdge { .. } => {
node_stage(
datastream,
"rewire_edge",
"skipped",
json!({"reason":"not implemented in myelin-worker image path"}),
);
Ok(())
}
StageCommandWire::ReleaseInputHandle { handle_id, .. } => { StageCommandWire::ReleaseInputHandle { handle_id, .. } => {
edge_runtime.release_input_handle(handle_id, worker, config, datastream, driver, stack) edge_runtime.release_input_handle(handle_id, worker, config, datastream, driver, stack)
} }

View file

@ -68,6 +68,13 @@ pub(crate) enum OrchestratorMsg {
stage_index: u32, stage_index: u32,
}, },
ObserveTokenEndpointsStopped, ObserveTokenEndpointsStopped,
ObserveOperatorStop {
run_id: u64,
},
ObserveMembershipLost {
run_id: u64,
node_id: u64,
},
AdvanceTimeMs(u64), AdvanceTimeMs(u64),
Snapshot { Snapshot {
reply_to: ActorAddress, reply_to: ActorAddress,
@ -131,7 +138,6 @@ pub(crate) enum RunCommandWire {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) enum LifecycleEventWire { pub(crate) enum LifecycleEventWire {
RunRejected { run_id: u64 },
RunFaulted { run_id: u64 }, RunFaulted { run_id: u64 },
RunCompleted { run_id: u64 }, RunCompleted { run_id: u64 },
RunOperatorStopped { run_id: u64 }, RunOperatorStopped { run_id: u64 },
@ -272,6 +278,17 @@ impl OrchestratorActor {
OrchestratorMsg::ObserveTokenEndpointsStopped => { OrchestratorMsg::ObserveTokenEndpointsStopped => {
self.core.observe(core::RunEvent::TokenEndpointsStopped) self.core.observe(core::RunEvent::TokenEndpointsStopped)
} }
OrchestratorMsg::ObserveOperatorStop { run_id } => {
self.core.observe(core::RunEvent::OperatorStop {
run_id: core::RunId(run_id),
});
}
OrchestratorMsg::ObserveMembershipLost { run_id, node_id } => {
self.core.observe(core::RunEvent::MembershipLost {
run_id: core::RunId(run_id),
node_id: core::NodeId(node_id),
});
}
OrchestratorMsg::AdvanceTimeMs(delta) => self.core.advance_time_ms(delta), OrchestratorMsg::AdvanceTimeMs(delta) => self.core.advance_time_ms(delta),
} }
} }
@ -449,9 +466,6 @@ impl From<&core::RunCommand> for RunCommandWire {
impl From<&core::LifecycleEvent> for LifecycleEventWire { impl From<&core::LifecycleEvent> for LifecycleEventWire {
fn from(event: &core::LifecycleEvent) -> Self { fn from(event: &core::LifecycleEvent) -> Self {
match event { match event {
core::LifecycleEvent::RunRejected { run_id, .. } => {
Self::RunRejected { run_id: run_id.0 }
}
core::LifecycleEvent::RunFaulted { run_id, .. } => { core::LifecycleEvent::RunFaulted { run_id, .. } => {
Self::RunFaulted { run_id: run_id.0 } Self::RunFaulted { run_id: run_id.0 }
} }

View file

@ -19,7 +19,7 @@ use crate::node_actor::{
#[cfg(feature = "dashboard")] #[cfg(feature = "dashboard")]
use crate::observability::dashboard_view::MyelinClusterDashboardView; use crate::observability::dashboard_view::MyelinClusterDashboardView;
use crate::observability::{benchmark, frame_archive::FrameArchive}; use crate::observability::{benchmark, frame_archive::FrameArchive};
use crate::orchestration::actor::{OrchestratorActor, OrchestratorReport}; use crate::orchestration::actor::{OrchestratorActor, OrchestratorMsg, OrchestratorReport};
use crate::orchestration::config::{DEFAULT_CONFIG_PATH, TomlConfigOverlay}; use crate::orchestration::config::{DEFAULT_CONFIG_PATH, TomlConfigOverlay};
const PROVIDER_START_MAX_ATTEMPTS: usize = 4; const PROVIDER_START_MAX_ATTEMPTS: usize = 4;
@ -56,6 +56,7 @@ use datastream::{
}; };
use distribution::node::DistributedNodeConfig; use distribution::node::DistributedNodeConfig;
use distribution::telemetry::{MembershipTransition, SwimProbeEvent}; use distribution::telemetry::{MembershipTransition, SwimProbeEvent};
use distribution::swim::telemetry::ObservedTransition;
use distribution::types::{MemberState, NodeId as DistNodeId}; use distribution::types::{MemberState, NodeId as DistNodeId};
use iroh::EndpointAddr; use iroh::EndpointAddr;
use iroh_driver::{ use iroh_driver::{
@ -283,6 +284,8 @@ where
"connectivity_preflight":"ready", "connectivity_preflight":"ready",
}), }),
); );
let actors_channel = orch_datastream.channel_by_name("runtime.actors");
let orch_stats_hook = orch_datastream.producer.stats_hook_on(actors_channel);
let stack = DistributionRuntimeStack::new_with_codecs( let stack = DistributionRuntimeStack::new_with_codecs(
driver.node_id(), driver.node_id(),
DistributedNodeConfig::default(), DistributedNodeConfig::default(),
@ -290,6 +293,7 @@ where
register_myelin_actor_codecs(registry); register_myelin_actor_codecs(registry);
datastream::wire::register_datastream_codec(registry); datastream::wire::register_datastream_codec(registry);
}, },
Some(orch_stats_hook),
); );
bootstrap( bootstrap(
&mut orch_datastream, &mut orch_datastream,
@ -455,7 +459,7 @@ where
tx: Mutex::new(obs_tx), tx: Mutex::new(obs_tx),
})); }));
let pipeline_coordinator_endpoint = coordinator_endpoint.clone(); let pipeline_coordinator_endpoint = coordinator_endpoint.clone();
let (mut provisioned_nodes, ready) = start_and_provision_workers( let (mut provisioned_nodes, ready, swim_to_node) = start_and_provision_workers(
provisioner, provisioner,
&config, &config,
pipeline_plan.as_ref(), pipeline_plan.as_ref(),
@ -473,6 +477,7 @@ where
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}, },
sink, sink,
coordinator_endpoint, coordinator_endpoint,
@ -547,6 +552,7 @@ where
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}, },
&work_rx, &work_rx,
&prompt_events, &prompt_events,
@ -558,6 +564,7 @@ where
tokenizer_reply_actor, tokenizer_reply_actor,
pipeline_plan.as_ref(), pipeline_plan.as_ref(),
ready.first_stage.endpoint.clone(), ready.first_stage.endpoint.clone(),
&swim_to_node,
); );
if let Err(error) = &result { if let Err(error) = &result {
bootstrap( bootstrap(
@ -2024,6 +2031,7 @@ struct RuntimeReadyAckLoop<'a> {
run_id: u64, run_id: u64,
orchestrator_node_id: u64, orchestrator_node_id: u64,
provider: &'a ProviderKind, provider: &'a ProviderKind,
orchestrator_actor: ActorAddress,
} }
fn wait_for_runtime_ready_acks( fn wait_for_runtime_ready_acks(
@ -2045,6 +2053,7 @@ fn wait_for_runtime_ready_acks(
run_id, run_id,
orchestrator_node_id, orchestrator_node_id,
provider, provider,
..
} = ctx; } = ctx;
let bootstrap = |ds: &mut OrchDatastream, phase: &str, status: &str, detail: Value| { let bootstrap = |ds: &mut OrchDatastream, phase: &str, status: &str, detail: Value| {
ds.emit_bootstrap( ds.emit_bootstrap(
@ -2232,7 +2241,7 @@ fn start_and_provision_workers(
coordinator: EndpointAddr, coordinator: EndpointAddr,
pipeline_coordinator: EndpointAddr, pipeline_coordinator: EndpointAddr,
orchestrator_actor: ActorAddress, orchestrator_actor: ActorAddress,
) -> Result<(ProvisionedClusterGuard, PromptRuntimeReady), String> { ) -> Result<(ProvisionedClusterGuard, PromptRuntimeReady, BTreeMap<DistNodeId, u64>), String> {
let RuntimeReadyAckLoop { let RuntimeReadyAckLoop {
driver, driver,
stack, stack,
@ -2439,6 +2448,7 @@ fn start_and_provision_workers(
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}, },
&expected_node_ids, &expected_node_ids,
) { ) {
@ -2468,6 +2478,7 @@ fn start_and_provision_workers(
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}) { }) {
Ok(ready) => ready, Ok(ready) => ready,
Err(error) => { Err(error) => {
@ -2512,6 +2523,7 @@ fn start_and_provision_workers(
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}, },
&ack_targets, &ack_targets,
&pipeline_coordinator, &pipeline_coordinator,
@ -2555,6 +2567,7 @@ fn start_and_provision_workers(
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}, },
expected_node_ids.len(), expected_node_ids.len(),
pipeline_plan.expect("pipeline mode requires plan"), pipeline_plan.expect("pipeline mode requires plan"),
@ -2578,6 +2591,7 @@ fn start_and_provision_workers(
run_id: config.run_id, run_id: config.run_id,
orchestrator_node_id: config.node_id, orchestrator_node_id: config.node_id,
provider: &config.provider, provider: &config.provider,
orchestrator_actor,
}, },
config.stage_index, config.stage_index,
) )
@ -2640,7 +2654,11 @@ fn start_and_provision_workers(
final_stage: ready, final_stage: ready,
} }
}; };
Ok((provisioned_nodes, prompt_ready)) let swim_to_node = readies
.iter()
.map(|(node_id, ready)| (ready.swim_node_id, *node_id))
.collect::<BTreeMap<DistNodeId, u64>>();
Ok((provisioned_nodes, prompt_ready, swim_to_node))
} }
fn stage_node_specs( fn stage_node_specs(
@ -2999,6 +3017,7 @@ fn wait_for_weights_loaded_count(
run_id, run_id,
orchestrator_node_id: node_id, orchestrator_node_id: node_id,
provider, provider,
..
} = ctx; } = ctx;
let expected_stages = pipeline_plan let expected_stages = pipeline_plan
.stages .stages
@ -4126,6 +4145,7 @@ fn wait_for_runtime_ready(ctx: RuntimeReadyAckLoop<'_>) -> Result<RuntimeReady,
run_id, run_id,
orchestrator_node_id: node_id, orchestrator_node_id: node_id,
provider, provider,
..
} = ctx; } = ctx;
let mut pending_ready: Option<RuntimeReady> = None; let mut pending_ready: Option<RuntimeReady> = None;
let mut node_swim_started = false; let mut node_swim_started = false;
@ -4271,6 +4291,7 @@ fn wait_for_weights_loaded(ctx: RuntimeReadyAckLoop<'_>, stage_index: u32) -> Re
run_id, run_id,
orchestrator_node_id: node_id, orchestrator_node_id: node_id,
provider, provider,
..
} = ctx; } = ctx;
loop { loop {
pump(driver, stack, frame_tx); pump(driver, stack, frame_tx);
@ -4913,6 +4934,7 @@ fn serve_prompts(
tokenizer_reply_to: ActorAddress, tokenizer_reply_to: ActorAddress,
pipeline_plan: Option<&run_plan::RunPlan>, pipeline_plan: Option<&run_plan::RunPlan>,
prompt_endpoint: EndpointAddr, prompt_endpoint: EndpointAddr,
swim_to_node: &BTreeMap<DistNodeId, u64>,
) -> Result<(), String> { ) -> Result<(), String> {
let RuntimeReadyAckLoop { let RuntimeReadyAckLoop {
driver, driver,
@ -4927,6 +4949,7 @@ fn serve_prompts(
run_id, run_id,
orchestrator_node_id: node_id, orchestrator_node_id: node_id,
provider, provider,
orchestrator_actor,
.. ..
} = ctx; } = ctx;
let emit_prompt_evt = let emit_prompt_evt =
@ -4949,6 +4972,7 @@ fn serve_prompts(
let mut active: Option<ActivePrompt> = None; let mut active: Option<ActivePrompt> = None;
loop { loop {
pump(driver, stack, frame_tx); pump(driver, stack, frame_tx);
orch_datastream.flush(dashboard, "orchestrator");
if let Some(pipeline) = pipeline_runtime.as_mut() { if let Some(pipeline) = pipeline_runtime.as_mut() {
pipeline.poll_driver(driver); pipeline.poll_driver(driver);
pipeline.drain_tokenizer_events( pipeline.drain_tokenizer_events(
@ -4971,6 +4995,21 @@ fn serve_prompts(
)?; )?;
drain_frames(frame_rx, dashboard, orch_datastream); drain_frames(frame_rx, dashboard, orch_datastream);
drain_orch_stdio_capture(orch_stdio_rx, orch_datastream, dashboard, run_id, node_id); drain_orch_stdio_capture(orch_stdio_rx, orch_datastream, dashboard, run_id, node_id);
let swim_transitions =
emit_swim_transitions(orch_datastream, dashboard, run_id, node_id, stack);
for transition in &swim_transitions {
if transition.to == MemberState::Dead
&& let Some(&lost_node_id) = swim_to_node.get(&transition.peer)
{
let _ = stack.runtime.send_to(
orchestrator_actor,
OrchestratorMsg::ObserveMembershipLost {
run_id,
node_id: lost_node_id,
},
);
}
}
if stop_rx.try_recv().is_ok() { if stop_rx.try_recv().is_ok() {
orch_datastream.emit_bootstrap( orch_datastream.emit_bootstrap(
dashboard, dashboard,
@ -4980,6 +5019,10 @@ fn serve_prompts(
"started", "started",
json!({"source":"stdin"}), json!({"source":"stdin"}),
); );
let _ = stack
.runtime
.send_to(orchestrator_actor, OrchestratorMsg::ObserveOperatorStop { run_id });
pump(driver, stack, frame_tx);
return Ok(()); return Ok(());
} }
@ -5337,8 +5380,9 @@ fn emit_swim_transitions(
run_id: u64, run_id: u64,
node_id: u64, node_id: u64,
stack: &DistributionRuntimeStack, stack: &DistributionRuntimeStack,
) { ) -> Vec<ObservedTransition> {
for transition in stack.drain_swim_transitions() { let transitions = stack.drain_swim_transitions();
for transition in &transitions {
let peer = format!("{:?}", transition.peer); let peer = format!("{:?}", transition.peer);
let from = transition.from.map(|state| format!("{:?}", state)); let from = transition.from.map(|state| format!("{:?}", state));
let to = format!("{:?}", transition.to); let to = format!("{:?}", transition.to);
@ -5366,8 +5410,9 @@ fn emit_swim_transitions(
"member_state":member_state.clone(), "member_state":member_state.clone(),
}), }),
); );
orch_datastream.emit_record(dashboard, &stack.membership_transition(&transition)); orch_datastream.emit_record(dashboard, &stack.membership_transition(transition));
} }
transitions
} }
fn emit_swim_probe_events( fn emit_swim_probe_events(
@ -5418,18 +5463,24 @@ fn local_tinygrad_worker_env(provider: &str) -> Option<(String, String)> {
} }
fn default_local_tinygrad_worker_path() -> Option<PathBuf> { fn default_local_tinygrad_worker_path() -> Option<PathBuf> {
let cwd_candidate = std::env::current_dir() // The tinygrad worker script ships in the node-image build context at
.ok() // `apps/myelin/node-image/tinygrad_worker.py` (see `chat/node_image.rs` and
.map(|cwd| cwd.join("apps").join("myelin-node").join("tinygrad_worker.py")); // the node-image Dockerfile). The process provider runs it directly via
// `python3`, so resolve that path from the workspace cwd or this crate's
// manifest dir. (Previously looked in `apps/myelin-node/`, a path left stale
// by the `mvp-system` -> `myelin` app refactor and never present on disk.)
let cwd_candidate = std::env::current_dir().ok().map(|cwd| {
cwd.join("apps")
.join("myelin")
.join("node-image")
.join("tinygrad_worker.py")
});
if let Some(candidate) = cwd_candidate.filter(|path| path.is_file()) { if let Some(candidate) = cwd_candidate.filter(|path| path.is_file()) {
return Some(candidate.canonicalize().unwrap_or(candidate)); return Some(candidate.canonicalize().unwrap_or(candidate));
} }
let manifest_candidate = PathBuf::from(env!("CARGO_MANIFEST_DIR")) let manifest_candidate = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..") .join("node-image")
.join("..")
.join("apps")
.join("myelin-node")
.join("tinygrad_worker.py"); .join("tinygrad_worker.py");
manifest_candidate.is_file().then(|| { manifest_candidate.is_file().then(|| {
manifest_candidate manifest_candidate

View file

@ -13,6 +13,7 @@ use std::time::{Duration, Instant};
use swactor::actor::{ActorAddress, ActorInterface}; use swactor::actor::{ActorAddress, ActorInterface};
use swactor::config::RuntimeConfig; use swactor::config::RuntimeConfig;
use swactor::runtime::{Ctx, Runtime}; use swactor::runtime::{Ctx, Runtime};
use swactor::stats::StatsHook;
use swactor::std::StdExtension; use swactor::std::StdExtension;
use swactor_transport::{CodecRegistry, CodecRemoteSink, NetworkMessage, TransportRouter}; use swactor_transport::{CodecRegistry, CodecRemoteSink, NetworkMessage, TransportRouter};
@ -44,7 +45,6 @@ pub(crate) struct DistributionActorAddrs {
} }
pub(crate) struct DistributionRuntimeStack { pub(crate) struct DistributionRuntimeStack {
pub node_id: NodeId,
pub runtime: Arc<Runtime>, pub runtime: Arc<Runtime>,
pub codec: Arc<CodecRegistry>, pub codec: Arc<CodecRegistry>,
pub outbox: Outbox, pub outbox: Outbox,
@ -61,6 +61,7 @@ impl DistributionRuntimeStack {
node_id: NodeId, node_id: NodeId,
config: DistributedNodeConfig, config: DistributedNodeConfig,
extend_codecs: impl FnOnce(&mut CodecRegistry), extend_codecs: impl FnOnce(&mut CodecRegistry),
stats_hook: Option<Arc<dyn StatsHook>>,
) -> Self { ) -> Self {
let mut runtime = let mut runtime =
Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new())); Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()));
@ -72,6 +73,9 @@ impl DistributionRuntimeStack {
Arc::clone(&codec), Arc::clone(&codec),
Arc::clone(&transport_router), Arc::clone(&transport_router),
))); )));
if let Some(hook) = stats_hook {
runtime.set_stats_hook(hook);
}
let runtime = Arc::new(runtime); let runtime = Arc::new(runtime);
let outbox: Outbox = Arc::new(Mutex::new(Vec::new())); let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
@ -146,7 +150,6 @@ impl DistributionRuntimeStack {
.expect("subscribe MembershipFanout"); .expect("subscribe MembershipFanout");
Self { Self {
node_id,
runtime, runtime,
codec, codec,
outbox, outbox,

View file

@ -72,12 +72,6 @@ impl StaticRelayProvider {
parse_relay_url(raw).map(Self::new) parse_relay_url(raw).map(Self::new)
} }
pub(crate) fn from_env() -> Result<Option<Self>, String> {
selected_relay_url_from_env()
.map(|url| Self::from_url_str(&url).map(Some))
.unwrap_or(Ok(None))
}
pub(crate) fn url(&self) -> String { pub(crate) fn url(&self) -> String {
self.url.to_string() self.url.to_string()
} }

View file

@ -16,16 +16,6 @@ pub(crate) struct RunPlan {
pub stages: Vec<StageRef>, pub stages: Vec<StageRef>,
} }
impl RunPlan {
pub(crate) fn test_linear(run_id: RunId, stages: Vec<StageRef>) -> Self {
Self { run_id, stages }
}
pub(crate) fn stage_nodes(&self) -> Vec<NodeId> {
self.stages.iter().map(|stage| stage.node_id).collect()
}
}
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct RunConfig { pub(crate) struct RunConfig {
pub run_id: RunId, pub run_id: RunId,
@ -125,10 +115,6 @@ pub(crate) enum RunFaultReason {
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum LifecycleEvent { pub(crate) enum LifecycleEvent {
RunRejected {
run_id: RunId,
reason: RunFaultReason,
},
RunFaulted { RunFaulted {
run_id: RunId, run_id: RunId,
reason: RunFaultReason, reason: RunFaultReason,
@ -175,8 +161,6 @@ pub(crate) enum RunCommand {
}, },
} }
pub type OrchestratorHarness = OrchestratorRun;
pub(crate) struct OrchestratorRun { pub(crate) struct OrchestratorRun {
config: RunConfig, config: RunConfig,
plan: Option<RunPlan>, plan: Option<RunPlan>,

View file

@ -165,7 +165,6 @@ pub(crate) enum EdgeKind {
pub(crate) enum ObjectKind { pub(crate) enum ObjectKind {
Token, Token,
Activation, Activation,
Weight,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]

View file

@ -19,12 +19,6 @@ pub(crate) struct DeviceHandle {
pub id: u64, pub id: u64,
} }
impl DeviceHandle {
pub(crate) fn new_current(id: u64) -> Self {
Self { generation: 1, id }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct LayerRange { pub(crate) struct LayerRange {
pub start: u32, pub start: u32,
@ -50,14 +44,6 @@ impl WeightSource {
tokenizer, tokenizer,
} }
} }
pub(crate) fn embedded_gguf(model_id: impl Into<String>, path: impl Into<String>) -> Self {
Self::new(
model_id,
GgufSource::LocalPath(path.into()),
TokenizerSource::EmbeddedGguf,
)
}
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -228,9 +214,6 @@ pub(crate) enum StageCommand {
range: LayerRange, range: LayerRange,
shard_plan: Option<StageShardPlan>, shard_plan: Option<StageShardPlan>,
}, },
RewireEdge {
edge_id: EdgeId,
},
ExecuteStep(ExecuteStep), ExecuteStep(ExecuteStep),
ReleaseInputHandle { ReleaseInputHandle {
object_id: ObjectId, object_id: ObjectId,
@ -244,8 +227,6 @@ pub(crate) enum StageCommand {
}, },
} }
pub type StageControllerHarness = StageController;
pub(crate) struct StageController { pub(crate) struct StageController {
local_node_id: NodeId, local_node_id: NodeId,
provision: Option<ProvisionStage>, provision: Option<ProvisionStage>,

View file

@ -0,0 +1,50 @@
//! Test-only constructors and harness type aliases for the `tests` tree.
//!
//! These helpers previously lived in the production modules
//! (`orchestration/run_fsm`, `staging::control`) but serve only test code.
//! Defining them here keeps them compiled under `#[cfg(test)]` only, so
//! production builds stay free of the dead-code warnings they would otherwise
//! trigger. Inherent methods resolve through their type, so existing call sites
//! such as `fsm::RunPlan::test_linear(..)` and
//! `stage::WeightSource::embedded_gguf(..)` work unchanged.
use crate::run_fsm::{NodeId as FsmNodeId, OrchestratorRun, RunId as FsmRunId, RunPlan, StageRef};
use crate::run_plan::{GgufSource, TokenizerSource};
use crate::staging::{DeviceHandle, StageController, WeightSource};
impl RunPlan {
/// Build a linear pipeline plan (stage 0 -> stage 1 -> ... -> last stage).
/// Test-only convenience over the public `RunPlan` fields.
pub(crate) fn test_linear(run_id: FsmRunId, stages: Vec<StageRef>) -> Self {
Self { run_id, stages }
}
/// Node ids in declared stage order.
pub(crate) fn stage_nodes(&self) -> Vec<FsmNodeId> {
self.stages.iter().map(|stage| stage.node_id).collect()
}
}
impl DeviceHandle {
/// A handle on the worker's current (first) generation.
pub(crate) fn new_current(id: u64) -> Self {
Self { generation: 1, id }
}
}
impl WeightSource {
/// Weights carried by an embedded GGUF file with an embedded tokenizer.
pub(crate) fn embedded_gguf(model_id: impl Into<String>, path: impl Into<String>) -> Self {
Self::new(
model_id,
GgufSource::LocalPath(path.into()),
TokenizerSource::EmbeddedGguf,
)
}
}
/// Test alias for the orchestrator run core, retained for readable test prose.
pub(crate) type OrchestratorHarness = OrchestratorRun;
/// Test alias for the stage controller core, retained for readable test prose.
pub(crate) type StageControllerHarness = StageController;

View file

@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
use crate::run_fsm as fsm; use crate::run_fsm as fsm;
use crate::run_plan as plan; use crate::run_plan as plan;
use crate::tests::harness::OrchestratorHarness;
use data_plane::edge_actor; use data_plane::edge_actor;
use myelin::observability::lifecycle as obs; use myelin::observability::lifecycle as obs;
use myelin::orchestration::engine_builder as engine; use myelin::orchestration::engine_builder as engine;
@ -34,7 +35,7 @@ pub struct LocalMockCluster {
engine_events: Vec<engine::EngineEvent>, engine_events: Vec<engine::EngineEvent>,
plan: plan::RunPlan, plan: plan::RunPlan,
nodes: BTreeMap<u32, MockNode>, nodes: BTreeMap<u32, MockNode>,
orchestrator: Option<fsm::OrchestratorHarness>, orchestrator: Option<OrchestratorHarness>,
orchestrator_command_cursor: usize, orchestrator_command_cursor: usize,
orchestrator_event_cursor: usize, orchestrator_event_cursor: usize,
trace: Vec<obs::Event>, trace: Vec<obs::Event>,
@ -303,7 +304,7 @@ impl LocalMockCluster {
self.orchestrator_command_cursor = 0; self.orchestrator_command_cursor = 0;
self.orchestrator_event_cursor = 0; self.orchestrator_event_cursor = 0;
self.scenario = scenario; self.scenario = scenario;
self.orchestrator = Some(fsm::OrchestratorHarness::new(fsm::RunConfig { self.orchestrator = Some(OrchestratorHarness::new(fsm::RunConfig {
run_id: fsm::RunId(self.run_id.0), run_id: fsm::RunId(self.run_id.0),
max_tokens: u64::from(self.max_tokens), max_tokens: u64::from(self.max_tokens),
prompt: tokenize(prompt), prompt: tokenize(prompt),
@ -742,7 +743,6 @@ impl LocalMockCluster {
self.push_run(obs::EventKind::RunCompleted, obs::Component::Orchestrator); self.push_run(obs::EventKind::RunCompleted, obs::Component::Orchestrator);
} }
fsm::LifecycleEvent::RunFaulted { .. } fsm::LifecycleEvent::RunFaulted { .. }
| fsm::LifecycleEvent::RunRejected { .. }
| fsm::LifecycleEvent::RunOperatorStopped { .. } => { | fsm::LifecycleEvent::RunOperatorStopped { .. } => {
self.push_run(obs::EventKind::RunFaulted, obs::Component::Orchestrator); self.push_run(obs::EventKind::RunFaulted, obs::Component::Orchestrator);
} }
@ -852,7 +852,7 @@ impl LocalMockCluster {
.count() .count()
} }
fn orchestrator_mut(&mut self) -> &mut fsm::OrchestratorHarness { fn orchestrator_mut(&mut self) -> &mut OrchestratorHarness {
self.orchestrator self.orchestrator
.as_mut() .as_mut()
.expect("run_prompt must initialize orchestrator") .expect("run_prompt must initialize orchestrator")

View file

@ -1,6 +1,7 @@
use crate::run_plan as plan; use crate::run_plan as plan;
use data_plane::edge_actor; use data_plane::edge_actor;
use myelin::staging as stage; use myelin::staging as stage;
use crate::tests::harness::StageControllerHarness;
use super::mock_transport::MockObject; use super::mock_transport::MockObject;
use super::mock_worker::MockWorker; use super::mock_worker::MockWorker;
@ -16,7 +17,7 @@ pub struct MockNode {
inbound_edge: Option<plan::EdgeId>, inbound_edge: Option<plan::EdgeId>,
outbound_edge: Option<plan::EdgeId>, outbound_edge: Option<plan::EdgeId>,
outbound_object_allocator: Option<edge_actor::ObjectIdAllocator>, outbound_object_allocator: Option<edge_actor::ObjectIdAllocator>,
controller: stage::StageControllerHarness, controller: StageControllerHarness,
worker: MockWorker, worker: MockWorker,
event_cursor: usize, event_cursor: usize,
} }
@ -34,7 +35,7 @@ impl MockNode {
inbound_edge: None, inbound_edge: None,
outbound_edge: None, outbound_edge: None,
outbound_object_allocator: None, outbound_object_allocator: None,
controller: stage::StageControllerHarness::new(stage::NodeId(node_id.0)), controller: StageControllerHarness::new(stage::NodeId(node_id.0)),
worker: MockWorker::new(stage_index, eos_after_sequence), worker: MockWorker::new(stage_index, eos_after_sequence),
event_cursor: 0, event_cursor: 0,
} }

View file

@ -1,3 +1,4 @@
mod harness;
mod local_e2e_guarantees; mod local_e2e_guarantees;
mod local_mock; mod local_mock;
mod node_guarantees; mod node_guarantees;

View file

@ -502,6 +502,7 @@ mod run_fsm {
//! `specs/BEHAVIOR_GUARANTEES.md`. //! `specs/BEHAVIOR_GUARANTEES.md`.
use crate::run_fsm as fsm; use crate::run_fsm as fsm;
use crate::tests::harness::OrchestratorHarness;
// A three-stage plan proves multi-stage provisioning and readiness without // A three-stage plan proves multi-stage provisioning and readiness without
// making tests depend on any placement heuristic. The plan is already valid; // making tests depend on any placement heuristic. The plan is already valid;
@ -529,8 +530,8 @@ mod run_fsm {
// The harness is the black-box public boundary for the run FSM. It accepts // The harness is the black-box public boundary for the run FSM. It accepts
// observable events and records emitted commands/events; tests never inspect an // observable events and records emitted commands/events; tests never inspect an
// internal FSM enum or private readiness counter. // internal FSM enum or private readiness counter.
fn new_run() -> fsm::OrchestratorHarness { fn new_run() -> OrchestratorHarness {
fsm::OrchestratorHarness::new(fsm::RunConfig { OrchestratorHarness::new(fsm::RunConfig {
run_id: fsm::RunId(7), run_id: fsm::RunId(7),
max_tokens: 4, max_tokens: 4,
prompt: vec![101, 102, 103], prompt: vec![101, 102, 103],
@ -680,7 +681,6 @@ mod run_fsm {
}); });
assert!(invalid.events().iter().any(|event| { assert!(invalid.events().iter().any(|event| {
matches!(event, fsm::LifecycleEvent::RunFaulted { .. }) matches!(event, fsm::LifecycleEvent::RunFaulted { .. })
|| matches!(event, fsm::LifecycleEvent::RunRejected { .. })
})); }));
} }

View file

@ -12,6 +12,7 @@ mod stage_controller {
//! `specs/BEHAVIOR_GUARANTEES.md`. //! `specs/BEHAVIOR_GUARANTEES.md`.
use myelin::staging as stage; use myelin::staging as stage;
use crate::tests::harness::StageControllerHarness;
// This provision fixture represents a single middle stage. It has one inbound // This provision fixture represents a single middle stage. It has one inbound
// and one outbound edge so tests can prove the controller uses assigned edges // and one outbound edge so tests can prove the controller uses assigned edges
@ -37,8 +38,8 @@ mod stage_controller {
// The harness exposes only public messages. Tests intentionally do not inspect // The harness exposes only public messages. Tests intentionally do not inspect
// private controller states such as "Preparing" or "Executing"; they infer // private controller states such as "Preparing" or "Executing"; they infer
// controller behavior from emitted commands and lifecycle events. // controller behavior from emitted commands and lifecycle events.
fn new_controller() -> stage::StageControllerHarness { fn new_controller() -> StageControllerHarness {
stage::StageControllerHarness::new(stage::NodeId(11)) StageControllerHarness::new(stage::NodeId(11))
} }
// Preparation readiness has four independent prerequisites. Listing them as // Preparation readiness has four independent prerequisites. Listing them as
@ -60,7 +61,7 @@ mod stage_controller {
// This helper provisions and readies a stage through public events. Tests that // This helper provisions and readies a stage through public events. Tests that
// focus on execution use it to avoid duplicating setup while still going through // focus on execution use it to avoid duplicating setup while still going through
// the same observable path as production. // the same observable path as production.
fn ready_stage() -> stage::StageControllerHarness { fn ready_stage() -> StageControllerHarness {
let mut harness = new_controller(); let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage { harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99), from: stage::NodeId(99),
@ -103,15 +104,6 @@ mod stage_controller {
) )
})); }));
// The controller must not emit any command that replaces the provisioned
// edge ids with a locally chosen edge.
assert!(
!harness
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) })
);
// An unauthorized provision attempt must fault before setup can begin. // An unauthorized provision attempt must fault before setup can begin.
let mut unauthorized = new_controller(); let mut unauthorized = new_controller();
unauthorized.observe(stage::StageEvent::ProvisionStage { unauthorized.observe(stage::StageEvent::ProvisionStage {

View file

@ -8,4 +8,4 @@ Keep this crate read-only with respect to observed programs.
- It must not send control signals to observed runtimes. - It must not send control signals to observed runtimes.
- It must not require changes outside `crates/dashboard` for dashboard-only work. - It must not require changes outside `crates/dashboard` for dashboard-only work.
Main built-in view: `/view/swactor/workers`, backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present. Main built-in views: `/view/swactor/workers` (worker-centric) and `/view/swactor/actor-overview` + `/view/swactor/actor-dossier` (actor-centric), all backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present. Actor views are pure frame consumers and tolerant of publisher shape.

View file

@ -10,5 +10,5 @@ swactor = { path = "../..", features = ["serde"] }
parking_lot = "0.12" parking_lot = "0.12"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
tokio = { version = "1", features = ["net", "rt-multi-thread", "sync"] } tokio.workspace = true
tokio-stream = "0.1" tokio-stream = "0.1"

View file

@ -2,7 +2,7 @@
Read-only HTML/SSE dashboard over incoming datastream frames. Read-only HTML/SSE dashboard over incoming datastream frames.
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor worker page is hosted here because worker/actor/message processing is universal to swactor programs. The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor views are hosted here because worker/actor/message processing is universal to swactor programs: the worker page, and the actor overview (fused roster) plus per-actor dossier.
## Routes ## Routes
@ -16,5 +16,9 @@ The crate owns the Axum server, bounded raw frame window, and view registry. Com
- `GET /api/view/fleet` — fleet and machine telemetry JSON snapshot - `GET /api/view/fleet` — fleet and machine telemetry JSON snapshot
- `GET /view/swactor/workers` — built-in worker page - `GET /view/swactor/workers` — built-in worker page
- `GET /api/view/swactor/workers` — worker page JSON snapshot - `GET /api/view/swactor/workers` — worker page JSON snapshot
- `GET /view/swactor/actor-overview` — built-in actor overview + roster page
- `GET /api/view/swactor/actor-overview` — actor overview JSON snapshot
- `GET /view/swactor/actor-dossier` — built-in per-actor dossier page
- `GET /api/view/swactor/actor-dossier` — actor dossier JSON snapshot
All state is derived from observed frames. The dashboard sends no control signals back to producers. All state is derived from observed frames. The dashboard sends no control signals back to producers.

View file

@ -97,6 +97,8 @@ impl DashboardHandle {
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default())); views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
views.register(Arc::new(hardware_view::HardwareDashboardView::default())); views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
views.register(swactor::worker_view()); views.register(swactor::worker_view());
views.register(swactor::actor_overview_view());
views.register(swactor::actor_dossier_view());
let store = Arc::new(DashboardStore::new( let store = Arc::new(DashboardStore::new(
config.raw_frame_history, config.raw_frame_history,
Arc::clone(&views), Arc::clone(&views),

View file

@ -0,0 +1,185 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Actor dossier — swactor</title>
<style>
:root{
--bg:#0f172a; --panel:#1e293b; --panel2:#172033; --border:#334155;
--text:#e2e8f0; --muted:#94a3b8; --accent:#60a5fa;
--running:#34d399; --poisoned:#f87171;
font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;
}
*{box-sizing:border-box}
body{margin:0;padding:0;background:var(--bg);color:var(--text);}
.mono,.table,th,td,.value,.num{font-variant-numeric:tabular-nums;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;}
header.bar{display:flex;align-items:center;gap:16px;padding:14px 20px;border-bottom:1px solid var(--border);background:var(--panel);position:sticky;top:0;z-index:5;}
header.bar h1{font-size:18px;margin:0;font-weight:600;letter-spacing:.01em;}
nav.tabs{display:flex;gap:4px;margin-left:auto;}
nav.tabs a{padding:6px 12px;border-radius:8px;color:var(--muted);text-decoration:none;font-size:13px;border:1px solid transparent;}
nav.tabs a:hover{color:var(--text);background:var(--panel2);}
nav.tabs a.cur{color:var(--accent);background:var(--panel2);border-color:var(--border);}
select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
.wrap{padding:20px;max-width:1100px;margin:0 auto;}
.picker{display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin-bottom:18px;}
.picker label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em;}
#pick{min-width:300px;max-width:560px;flex:1;}
.back-link{font-size:13px;}
.badge{display:inline-block;padding:2px 8px;border-radius:99px;font-size:11px;text-transform:uppercase;letter-spacing:.06em;font-weight:600;}
.empty{padding:28px;border:1px dashed #475569;border-radius:12px;color:var(--muted);text-align:center;}
.toast{position:fixed;left:50%;bottom:20px;transform:translateX(-50%);background:var(--panel2);border:1px solid var(--border);color:var(--text);padding:10px 16px;border-radius:10px;font-size:13px;max-width:min(90vw,560px);text-align:center;z-index:50;box-shadow:0 8px 24px rgba(0,0,0,.45);}
.toast.err{border-color:var(--poisoned);color:var(--poisoned);}
.kvs{display:grid;grid-template-columns:max-content 1fr;gap:6px 14px;}
.kvs .k{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.05em;}
.kvs .v{font-size:13px;word-break:break-word;}
.section{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:16px 18px;margin-bottom:16px;}
h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:0 0 12px;}
.mb-dyn{display:flex;gap:24px;align-items:center;flex-wrap:wrap;}
.mb-spark{flex:1 1 320px;min-width:0;}
.mb-spark .spark{display:block;width:100%;height:120px;}
.mb-vals{display:flex;flex-direction:column;gap:16px;min-width:150px;}
.mb-val .label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em;}
.mb-val .value{font-size:28px;margin-top:4px;font-weight:600;line-height:1;}
.bar-row{display:flex;align-items:center;gap:8px;margin:5px 0;}
.bar-row .name{width:48%;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}
.bar-track{flex:1;height:10px;background:var(--panel2);border-radius:99px;overflow:hidden;}
.bar-fill{height:100%;background:var(--accent);border-radius:99px;}
.bar-row .num{width:90px;text-align:right;color:var(--muted);}
.bar-row.last .bar-fill{background:var(--running);}
.bar-row.last{font-weight:600;}
a{color:var(--accent);text-decoration:none;}
.stale{opacity:.55;}
</style>
</head>
<body>
<header class="bar">
<h1>Actor dossier</h1>
<select id="runtime"></select>
<nav class="tabs">
<a href="/view/swactor/actor-overview">Overview</a>
<a href="/view/swactor/actor-dossier" class="cur">Dossier</a>
<a href="/view/swactor/workers">Workers</a>
</nav>
</header>
<div class="wrap">
<div class="picker">
<label for="pick">Actor</label>
<select id="pick"></select>
<a class="back-link" href="/view/swactor/actor-overview">&larr; back to overview</a>
</div>
<div id="empty" class="toast" hidden>No runtime actor frames yet.</div>
<div id="content"></div>
</div>
<script>
const POLL_MS=750;
const STATE_COLOR={running:'#34d399',poisoned:'#f87171'};
const elRuntime=document.getElementById('runtime');
const elPick=document.getElementById('pick');
const elEmpty=document.getElementById('empty');
const elContent=document.getElementById('content');
let runtimes=[],runtimeKey='',selectedAddr='';
function fmt(n,d=0){if(n==null||n===undefined)return'\u2014';if(typeof n!=='number')return String(n);return n.toLocaleString(undefined,{maximumFractionDigits:d});}
function esc(s){return String(s==null?'':s).replace(/[&<>"]/g,function(c){return{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c];});}
function shortAddr(a){a=String(a||'');return a.length<=12?a:a.slice(0,6)+'\u2026'+a.slice(-4);}
function urlActor(){try{return new URLSearchParams(location.search).get('actor');}catch(e){return null;}}
function stateBadge(a){const s=a.poisoned?'poisoned':'running';const c=STATE_COLOR[s];return '<span class="badge" style="color:'+c+';background:'+c+'22">'+s+'</span>';}
function growthColor(g){if(g>1.5)return '#f87171';if(g>0.3)return '#fbbf24';return '#34d399';}
function areaChart(data,w,h,color){
if(!data||data.length<2)return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="100%" height="'+h+'" preserveAspectRatio="none"></svg>';
const max=Math.max(1,...data);const n=data.length;
let line='',area='0,'+h+' ';
for(let i=0;i<n;i++){const x=(i/(n-1)*w).toFixed(1);const y=(h-(data[i]/max)*(h-2)-1).toFixed(1);line+=x+','+y+' ';area+=x+','+y+' ';}
area+=' '+w+','+h;
return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="100%" height="'+h+'" preserveAspectRatio="none">'+
'<polygon points="'+area.trim()+'" fill="'+(color||'#60a5fa')+'22"/>'+
'<polyline points="'+line.trim()+'" fill="none" stroke="'+(color||'#60a5fa')+'" stroke-width="1.5"/></svg>';
}
function kv(k,vhtml){return '<div class="k">'+esc(k)+'</div><div class="v">'+vhtml+'</div>';}
function typeShort(t){t=String(t||'');const i=t.lastIndexOf('::');return i>=0?t.slice(i+2):t;}
function pickRuntime(list){
if(!runtimeKey||!list.some(r=>r.stream.key===runtimeKey)){
const live=list.find(r=>r.live)||list[0];runtimeKey=live?live.stream.key:'';
}
return list.find(r=>r.stream.key===runtimeKey)||null;
}
function currentActors(rt){return rt?(rt.actors||[]):[];}
function findActor(rt){
const actors=currentActors(rt);
if(selectedAddr){const hit=actors.find(a=>a.address===selectedAddr);if(hit)return hit;}
return actors[0]||null;
}
function renderPicker(rt){
const actors=currentActors(rt);
elPick.innerHTML=actors.map(a=>
'<option value="'+esc(a.address)+'">'+esc(shortAddr(a.address))+(a.name?' \u00b7 '+esc(a.name):'')+'</option>'
).join('');
const cur=findActor(rt);
if(cur){selectedAddr=cur.address;elPick.value=selectedAddr;}
}
function renderRuntimeSelector(){
elRuntime.innerHTML=runtimes.map(r=>
'<option value="'+esc(r.stream.key)+'">'+esc(r.stream.node)+'#'+r.stream.life+(r.live?'':' (stale)')+'</option>'
).join('');
if(runtimeKey)elRuntime.value=runtimeKey;
}
function render(a,rt){
if(!a){elContent.innerHTML='<div class="empty">Select an actor. None are present on the current frame.</div>';return;}
const lastType=a.last_msg_type||'\u2014';
const diet=(a.message_type_counts||[]).slice();
diet.sort((x,y)=>y.count-x.count);
const dmax=Math.max(1,...diet.map(d=>d.count));
const lastName=a.last_msg_type;
const dietHtml=diet.length?diet.map(d=>{
const cls=d.message_type===lastName?'bar-row last':'bar-row';
return '<div class="'+cls+'"><span class="name">'+esc(d.message_type)+'</span>'+
'<div class="bar-track"><div class="bar-fill" style="width:'+(d.count/dmax*100).toFixed(1)+'%"></div></div>'+
'<span class="num">'+fmt(d.count)+'</span></div>';
}).join(''):'<div class="empty">no message-type counts on frame</div>';
const hist=(a.history||[]).map(h=>h.mailbox_depth);
const g=a.mailbox_growth||0;
elContent.innerHTML=
'<div class="section"><h2>Identity &amp; lifecycle</h2><div class="kvs">'+
kv('address','<span class="mono">'+esc(a.address)+'</span>')+
(a.name?kv('name',esc(a.name)):'')+
(a.actor_type?kv('actor type','<span class="mono" title="'+esc(a.actor_type)+'">'+esc(typeShort(a.actor_type))+'</span>'):'')+
(a.message_type?kv('message type','<span class="mono" title="'+esc(a.message_type)+'">'+esc(typeShort(a.message_type))+'</span>'):'')+
kv('state',stateBadge(a))+
kv('worker',a.worker_id==null?'\u2014':esc(a.worker_id))+
kv('last message',esc(lastType))+
kv('last seen',a.last_seen_ms_ago!=null?(a.last_seen_ms_ago/1000).toFixed(1)+'s ago':'\u2014')+
'</div></div>'+
'<div class="section"><h2>Mailbox dynamics</h2><div class="mb-dyn">'+
'<div class="mb-spark">'+areaChart(hist,720,120,growthColor(g))+'</div>'+
'<div class="mb-vals">'+
'<div class="mb-val"><div class="label">depth</div><div class="value" style="color:'+growthColor(g)+'">'+fmt(a.mailbox_depth)+'</div></div>'+
'<div class="mb-val"><div class="label">growth</div><div class="value">'+fmt(g,1)+'/s</div></div>'+
'<div class="mb-val"><div class="label">msg/s</div><div class="value">'+fmt(a.msg_per_sec,1)+'</div></div>'+
'</div>'+
'</div></div>'+
'<div class="section"><h2>Message diet</h2>'+dietHtml+'</div>';
}
async function refresh(){
try{
const res=await fetch('/api/view/swactor/actor-dossier',{cache:'no-store'});
if(!res.ok)throw new Error('HTTP '+res.status);
const data=await res.json();
runtimes=data.runtimes||[];
renderRuntimeSelector();
const rt=pickRuntime(runtimes);
if(rt&&currentActors(rt).length>0){elEmpty.hidden=true;}else{elEmpty.className='toast';elEmpty.textContent='No runtime actor frames yet.';elEmpty.hidden=false;}
renderPicker(rt);
render(findActor(rt),rt);
}catch(e){
elEmpty.className='toast err';elEmpty.hidden=false;elEmpty.textContent='Failed to load actor frames: '+e.message;elContent.innerHTML='';
}
}
elRuntime.addEventListener('change',e=>{runtimeKey=e.target.value;const rt=pickRuntime(runtimes);renderPicker(rt);render(findActor(rt),rt);});
elPick.addEventListener('change',e=>{selectedAddr=e.target.value;const rt=pickRuntime(runtimes);render(findActor(rt),rt);});
const q=urlActor();if(q)selectedAddr=q;
refresh();
setInterval(refresh,POLL_MS);
</script>
</body>
</html>

View file

@ -0,0 +1,312 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Actor overview — swactor</title>
<style>
:root{
--bg:#0f172a; --panel:#1e293b; --panel2:#172033; --border:#334155;
--text:#e2e8f0; --muted:#94a3b8; --accent:#60a5fa;
--running:#34d399; --poisoned:#f87171;
font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;
}
*{box-sizing:border-box}
body{margin:0;padding:0;background:var(--bg);color:var(--text);}
.mono,.table,th,td,.value,.num{font-variant-numeric:tabular-nums;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;}
header.bar{display:flex;align-items:center;gap:16px;padding:14px 20px;border-bottom:1px solid var(--border);background:var(--panel);position:sticky;top:0;z-index:5;}
header.bar h1{font-size:18px;margin:0;font-weight:600;letter-spacing:.01em;}
nav.tabs{display:flex;gap:4px;margin-left:auto;}
nav.tabs a{padding:6px 12px;border-radius:8px;color:var(--muted);text-decoration:none;font-size:13px;border:1px solid transparent;}
nav.tabs a:hover{color:var(--text);background:var(--panel2);}
nav.tabs a.cur{color:var(--accent);background:var(--panel2);border-color:var(--border);}
select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
.wrap{padding:20px;max-width:1400px;margin:0 auto;}
.grid{display:grid;gap:12px;}
.cards{grid-template-columns:repeat(auto-fit,minmax(150px,1fr));margin-bottom:16px;}
.card{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:14px;}
.card .label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em;}
.card .value{font-size:26px;margin-top:6px;font-weight:600;}
.card .value.bad{color:var(--poisoned);}
.badge{display:inline-block;padding:2px 8px;border-radius:99px;font-size:11px;text-transform:uppercase;letter-spacing:.06em;font-weight:600;}
.table{width:100%;border-collapse:collapse;background:var(--panel);border:1px solid var(--border);border-radius:12px;overflow:hidden;}
.table th,.table td{padding:8px 10px;border-bottom:1px solid #1f2937;text-align:left;white-space:nowrap;}
.table th{color:#93c5fd;background:var(--panel2);cursor:pointer;user-select:none;font-size:12px;text-transform:uppercase;letter-spacing:.05em;}
.table th:hover{color:#fff;}
.table tr{cursor:pointer;}
.table tbody tr:hover{background:#1d4ed822;}
.table td.muted,.muted{color:var(--muted);}
.spark{display:block;}
.bar-row{display:flex;align-items:center;gap:8px;margin:5px 0;}
.bar-row .name{width:38%;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}
.bar-track{flex:1;height:10px;background:var(--panel2);border-radius:99px;overflow:hidden;}
.bar-fill{height:100%;background:var(--accent);border-radius:99px;}
.bar-row .num{width:90px;text-align:right;color:var(--muted);}
input{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
.filters{display:flex;gap:10px;flex-wrap:wrap;align-items:center;margin-bottom:14px;}
.filters label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em;}
h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:22px 0 10px;}
.empty{padding:28px;border:1px dashed #475569;border-radius:12px;color:var(--muted);text-align:center;}
.toast{position:fixed;left:50%;bottom:20px;transform:translateX(-50%);background:var(--panel2);border:1px solid var(--border);color:var(--text);padding:10px 16px;border-radius:10px;font-size:13px;max-width:min(90vw,560px);text-align:center;z-index:50;box-shadow:0 8px 24px rgba(0,0,0,.45);}
.toast.err{border-color:var(--poisoned);color:var(--poisoned);}
.stacked{display:flex;height:18px;border-radius:6px;overflow:hidden;border:1px solid var(--border);}
.legend{display:flex;gap:14px;flex-wrap:wrap;margin-top:8px;font-size:12px;color:var(--muted);}
.legend i{display:inline-block;width:12px;height:12px;margin-right:5px;border-radius:3px;vertical-align:middle;}
a{color:var(--accent);text-decoration:none;}
.roster-box{max-height:520px;overflow:auto;border:1px solid var(--border);border-radius:12px;}
.roster-box .table{border-radius:0;border:none;}
.stale{opacity:.55;}
</style>
</head>
<body>
<header class="bar">
<h1>Actor overview</h1>
<select id="runtime"></select>
<nav class="tabs">
<a href="/view/swactor/actor-overview" class="cur">Overview</a>
<a href="/view/swactor/actor-dossier">Dossier</a>
<a href="/view/swactor/workers">Workers</a>
</nav>
</header>
<div class="wrap">
<div id="empty" class="toast" hidden>No runtime actor frames yet. A live swactor runtime publishing <code>runtime.actors</code> will populate this page.</div>
<div id="cards" class="grid cards"></div>
<h2>State distribution</h2>
<div class="stacked" id="dist"></div>
<div class="legend" id="distLegend"></div>
<h2>Throughput</h2>
<div id="throughput"></div>
<h2>Worker balance</h2>
<div id="workers"></div>
<h2>Hot actors — mailbox growth</h2>
<div id="hot"></div>
<h2>Alerts</h2>
<div id="alerts"></div>
<h2>Roster</h2>
<div class="filters">
<label for="f-search">search</label>
<input id="f-search" type="search" placeholder="name · addr · worker" autocomplete="off" spellcheck="false">
<label for="f-state">state</label>
<select id="f-state">
<option value="all">all states</option>
<option value="running">running</option>
<option value="poisoned">poisoned</option>
</select>
<label for="f-worker">worker</label>
<select id="f-worker"></select>
</div>
<div class="roster-box">
<table class="table">
<thead><tr id="head"></tr></thead>
<tbody id="rows"></tbody>
</table>
</div>
</div>
<script>
const POLL_MS=750;
const elRuntime=document.getElementById('runtime');
const elEmpty=document.getElementById('empty');
const elCards=document.getElementById('cards');
const elDist=document.getElementById('dist');
const elDistLegend=document.getElementById('distLegend');
const elThroughput=document.getElementById('throughput');
const elWorkers=document.getElementById('workers');
const elHot=document.getElementById('hot');
const elAlerts=document.getElementById('alerts');
const elHead=document.getElementById('head');
const elRows=document.getElementById('rows');
const elSearch=document.getElementById('f-search');
const elState=document.getElementById('f-state');
const elWorker=document.getElementById('f-worker');
const STATE_COLOR={running:'#34d399',poisoned:'#f87171'};
const COLS=[['actor','Actor'],['type','Type'],['state','State'],['mailbox','Mailbox'],['rate','Msg/s'],
['processed','Processed'],['last','Last msg'],['worker','Worker']];
const DEFDIR={actor:'asc',type:'asc',state:'asc',mailbox:'desc',rate:'desc',processed:'desc',last:'asc',worker:'asc'};
let sortKey='mailbox',sortDir='desc';
let fSearch='',fState='all',fWorker='all';
let runtimes=[],runtimeKey='';
function fmt(n,d=0){if(n==null||n===undefined)return'\u2014';if(typeof n!=='number')return String(n);return n.toLocaleString(undefined,{maximumFractionDigits:d});}
function ago(sec){sec=Math.max(0,Math.round(sec));if(sec<60)return sec+'s';const m=Math.floor(sec/60);if(m<60)return m+'m';return Math.floor(m/60)+'h';}
function esc(s){return String(s==null?'':s).replace(/[&<>"]/g,function(c){return{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c];});}
function shortAddr(a){a=String(a||'');return a.length<=12?a:a.slice(0,6)+'\u2026'+a.slice(-4);}
function typeShort(t){t=String(t||'');const i=t.lastIndexOf('::');return i>=0?t.slice(i+2):t;}
function stateBadge(a){const s=a.poisoned?'poisoned':'running';const c=STATE_COLOR[s];return '<span class="badge" style="color:'+c+';background:'+c+'22">'+s+'</span>';}
function growthColor(g){if(g>1.5)return '#f87171';if(g>0.3)return '#fbbf24';return '#34d399';}
function sparkline(data,w,h,color){
if(!data||data.length<2)return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="'+w+'" height="'+h+'"></svg>';
const max=Math.max(1,...data);const n=data.length;
let pts='';for(let i=0;i<n;i++)pts+=(i/(n-1)*w).toFixed(1)+','+(h-(data[i]/max)*(h-2)-1).toFixed(1)+' ';
return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="'+w+'" height="'+h+'" preserveAspectRatio="none"><polyline points="'+pts.trim()+'" fill="none" stroke="'+(color||'#60a5fa')+'" stroke-width="1.5"/></svg>';
}
function pickRuntime(list){
if(!runtimeKey||!list.some(r=>r.stream.key===runtimeKey)){
const live=list.find(r=>r.live)||list[0];
runtimeKey=live?live.stream.key:'';
}
return list.find(r=>r.stream.key===runtimeKey)||null;
}
function renderRuntimeSelector(){
elRuntime.innerHTML=runtimes.map(r=>
'<option value="'+esc(r.stream.key)+'">'+esc(r.stream.node)+'#'+r.stream.life+(r.live?'':' (stale)')+'</option>'
).join('');
if(runtimeKey)elRuntime.value=runtimeKey;
}
function renderCards(rt){
const s=rt.summary||{};
const uptime=s.uptime_ms!=null?ago(s.uptime_ms/1000):'\u2014';
elCards.innerHTML=
'<div class="card"><div class="label">Actors</div><div class="value">'+fmt(s.actors)+'</div></div>'+
'<div class="card"><div class="label">msg/s</div><div class="value">'+fmt(s.msg_per_sec,1)+'</div></div>'+
'<div class="card"><div class="label">Mailbox</div><div class="value">'+fmt(s.mailbox_depth)+'</div></div>'+
'<div class="card"><div class="label">Poisoned</div><div class="value'+(s.poisoned>0?' bad':'')+'">'+fmt(s.poisoned)+'</div></div>'+
'<div class="card"><div class="label">Uptime</div><div class="value">'+uptime+'</div></div>';
}
function renderDist(rt){
const actors=rt.actors||[];const n=actors.length||1;
let running=0,poisoned=0;
actors.forEach(a=>{if(a.poisoned)poisoned++;else running++;});
let segs='';
if(running)segs+='<div style="width:'+(running/n*100).toFixed(2)+'%;background:'+STATE_COLOR.running+';"></div>';
if(poisoned)segs+='<div style="width:'+(poisoned/n*100).toFixed(2)+'%;background:'+STATE_COLOR.poisoned+';"></div>';
elDist.innerHTML=segs||'<div style="width:100%;background:var(--panel2);"></div>';
elDistLegend.innerHTML='<span><i style="background:'+STATE_COLOR.running+'"></i>running: '+running+'</span>'+
'<span><i style="background:'+STATE_COLOR.poisoned+'"></i>poisoned: '+poisoned+'</span>';
}
function renderThroughput(rt){
const h=(rt.history||[]).map(x=>x.msg_per_sec);
const mb=(rt.history||[]).map(x=>x.mailbox_depth);
elThroughput.innerHTML=
sparkline(h,760,90,'#60a5fa')+'<div style="height:6px"></div>'+
sparkline(mb,760,90,'#fbbf24')+
'<div class="legend"><span><i style="background:#60a5fa"></i>msg/s</span><span><i style="background:#fbbf24"></i>mailbox</span></div>';
}
function renderWorkers(rt){
const counts={};
(rt.actors||[]).forEach(a=>{const w=a.worker_id==null?'?':a.worker_id;counts[w]=(counts[w]||0)+1;});
const keys=Object.keys(counts).sort();
const max=Math.max(1,...Object.values(counts));
elWorkers.innerHTML=keys.map(w=>{
const c=counts[w];
return '<div class="bar-row"><span class="name">worker '+esc(w)+'</span>'+
'<div class="bar-track"><div class="bar-fill" style="width:'+(c/max*100).toFixed(1)+'%"></div></div>'+
'<span class="num">'+c+'</span></div>';
}).join('')||'<div class="empty">no worker placement on frame</div>';
}
function renderHot(rt){
const hot=(rt.actors||[]).slice().sort((a,b)=>(b.mailbox_growth||0)-(a.mailbox_growth||0)).slice(0,8);
const gmax=Math.max(1,...hot.map(a=>Math.max(0,a.mailbox_growth||0)));
elHot.innerHTML=hot.map(a=>{
const g=a.mailbox_growth||0;const gc=growthColor(g);
return '<div class="bar-row" data-addr="'+esc(a.address)+'">'+
'<span class="name">'+shortAddr(a.address)+(a.name?' '+esc(a.name):'')+'</span>'+
'<div class="bar-track"><div class="bar-fill" style="width:'+(Math.max(0,g)/gmax*100).toFixed(1)+'%;background:'+gc+'"></div></div>'+
'<span class="num">'+fmt(g,1)+'/s</span></div>';
}).join('')||'<div class="empty">no actors</div>';
}
function renderAlerts(rt){
const alerted=(rt.actors||[]).filter(a=>a.poisoned||(a.mailbox_growth||0)>1.5);
if(!alerted.length){elAlerts.innerHTML='<div class="empty">fleet healthy</div>';return;}
alerted.sort((a,b)=>{const ap=a.poisoned?0:1,bp=b.poisoned?0:1;if(ap!==bp)return ap-bp;return (b.mailbox_growth||0)-(a.mailbox_growth||0);});
elAlerts.innerHTML='<table class="table"><tbody>'+
alerted.map(a=>{
const reason=a.poisoned?'poisoned':'saturating';
return '<tr data-addr="'+esc(a.address)+'"><td>'+shortAddr(a.address)+(a.name?' '+esc(a.name):'')+' '+stateBadge(a)+'</td>'+
'<td class="muted">'+reason+'</td><td style="text-align:right;color:var(--muted);width:1%;">\u2192</td></tr>';
}).join('')+'</tbody></table>';
}
function colValue(a,key){
switch(key){
case 'actor':return (a.name||a.address).toLowerCase();
case 'type':return typeShort(a.actor_type).toLowerCase();
case 'state':return a.poisoned?'poisoned':'running';
case 'mailbox':return a.mailbox_depth||0;
case 'rate':return a.msg_per_sec||0;
case 'processed':return a.messages_processed||0;
case 'last':return a.last_msg_type||'';
case 'worker':return a.worker_id==null?-1:a.worker_id;
}
return 0;
}
function visible(rt){
let list=rt.actors||[];
if(fSearch){const q=fSearch.toLowerCase();list=list.filter(a=>((a.name?a.name+' ':'')+(a.actor_type?typeShort(a.actor_type)+' '+a.actor_type+' ':'')+a.address+' '+(a.worker_id==null?'':a.worker_id)).toLowerCase().indexOf(q)>=0);}
if(fState!=='all')list=list.filter(a=>(a.poisoned?'poisoned':'running')===fState);
if(fWorker!=='all')list=list.filter(a=>String(a.worker_id)===fWorker);
const k=sortKey,dir=sortDir==='asc'?1:-1;
return list.slice().sort((x,y)=>{const vx=colValue(x,k),vy=colValue(y,k);let r;if(typeof vx==='number'&&typeof vy==='number')r=vx-vy;else r=String(vx).localeCompare(String(vy));return r*dir;});
}
function renderHead(){
elHead.innerHTML=COLS.map(c=>{
const key=c[0],label=c[1];
const arrow=sortKey===key?(sortDir==='asc'?' \u25B2':' \u25BC'):'';
const order=sortKey===key?(sortDir==='asc'?'ascending':'descending'):'none';
return '<th data-key="'+key+'" scope="col" aria-sort="'+order+'">'+label+arrow+'</th>';
}).join('');
}
function populateWorkerFilter(rt){
const seen={};(rt.actors||[]).forEach(a=>{if(a.worker_id!=null)seen[a.worker_id]=1;});
const keys=Object.keys(seen).sort((a,b)=>a-b);
elWorker.innerHTML='<option value="all">all workers</option>'+keys.map(w=>'<option value="'+esc(w)+'">'+esc(w)+'</option>').join('');
if(fWorker!=='all'&&!keys.includes(fWorker))fWorker='all';
elWorker.value=fWorker;
}
function renderRows(rt){
const list=visible(rt);
if(!list.length){elRows.innerHTML='<tr><td colspan="'+COLS.length+'"><div class="empty">No actors match the current filters.</div></td></tr>';return;}
elRows.innerHTML=list.map(a=>{
const hist=(a.history||[]).map(h=>h.mailbox_depth);
const gc=growthColor(a.mailbox_growth||0);
const mbox='<span style="display:inline-flex;align-items:center;gap:8px">'+
'<span style="color:'+gc+';font-weight:600">'+fmt(a.mailbox_depth)+'</span>'+sparkline(hist,60,16,gc)+'</span>';
const actorCell=shortAddr(a.address)+(a.name?'<br><span class="muted" style="font-size:11px">'+esc(a.name)+'</span>':'');
const typeCell=a.actor_type?'<span class="muted" title="'+esc(a.actor_type)+'">'+esc(typeShort(a.actor_type))+'</span>':'<span class="muted">\u2014</span>';
return '<tr data-addr="'+esc(a.address)+'">'+
'<td>'+actorCell+'</td><td>'+typeCell+'</td><td>'+stateBadge(a)+'</td><td>'+mbox+'</td>'+
'<td>'+fmt(a.msg_per_sec,1)+'</td><td>'+fmt(a.messages_processed)+'</td>'+
'<td><span class="muted" style="display:inline-block;max-width:240px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom">'+esc(a.last_msg_type)+'</span></td>'+
'<td>'+(a.worker_id==null?'<span class="muted">\u2014</span>':a.worker_id)+'</td></tr>';
}).join('');
}
function render(rt){
if(!rt){[elCards,elDist,elThroughput,elWorkers,elHot,elAlerts,elRows].forEach(e=>e.innerHTML='');return;}
elEmpty.hidden=true;
renderCards(rt);renderDist(rt);renderThroughput(rt);renderWorkers(rt);renderHot(rt);renderAlerts(rt);
populateWorkerFilter(rt);renderHead();renderRows(rt);
}
async function refresh(){
try{
const res=await fetch('/api/view/swactor/actor-overview',{cache:'no-store'});
if(!res.ok)throw new Error('HTTP '+res.status);
const data=await res.json();
runtimes=data.runtimes||[];
renderRuntimeSelector();
const rt=pickRuntime(runtimes);
if(rt){elEmpty.hidden=true;}else{elEmpty.className='toast';elEmpty.textContent='No runtime actor frames yet. A live swactor runtime publishing runtime.actors will populate this page.';elEmpty.hidden=false;}
render(rt);
}catch(e){
elEmpty.className='toast err';
elEmpty.textContent='Failed to load actor frames: '+e.message;
elEmpty.hidden=false;
}
}
function setSort(key){if(sortKey===key)sortDir=sortDir==='asc'?'desc':'asc';else{sortKey=key;sortDir=DEFDIR[key]||'asc';}renderHead();renderRows(pickRuntime(runtimes));}
elRuntime.addEventListener('change',e=>{runtimeKey=e.target.value;render(pickRuntime(runtimes));});
elSearch.addEventListener('input',e=>{fSearch=e.target.value.toLowerCase();renderRows(pickRuntime(runtimes));});
elState.addEventListener('change',e=>{fState=e.target.value;renderRows(pickRuntime(runtimes));});
elWorker.addEventListener('change',e=>{fWorker=e.target.value;renderRows(pickRuntime(runtimes));});
elHead.addEventListener('click',e=>{const th=e.target.closest('th[data-key]');if(th)setSort(th.getAttribute('data-key'));});
document.addEventListener('click',e=>{const el=e.target.closest('[data-addr]');if(el)location.href='/view/swactor/actor-dossier?actor='+encodeURIComponent(el.getAttribute('data-addr'));});
renderHead();
refresh();
setInterval(refresh,POLL_MS);
</script>
</body>
</html>

View file

@ -0,0 +1,635 @@
//! Actor-centric read-only view over `runtime.actors` / `runtime.stats` frames.
//!
//! Pure frame consumer: it folds incoming per-actor snapshots into local state
//! and exposes JSON + HTML. It sends nothing back to observed runtimes. Parsing
//! is intentionally tolerant of publisher shape — a frame is a flat record that
//! may come from the per-worker `DatastreamStatsHook` envelope
//! (`{worker_id, actors:[...]}`) or from a process that publishes a merged
//! `{actors:[...]}` payload (the dashboard dummy node, app runtimes). Fields the
//! publisher includes (name, worker_id, lifecycle flags) are displayed; ones it
//! omits are left blank rather than fabricated.
//!
//! Reliably on the frame today: address, mailbox depth, throughput, last
//! message, poisoned flag, and the message-type diet. `worker_id` and `name`
//! appear when the publisher sends them. Finer lifecycle granularity
//! (new/suspended/stopping), actor Rust type, and spawn age are not on the
//! current frame and are therefore not shown — enriching the feed is a separate
//! core concern, not a dashboard one.
//!
//! Two pages share one data model: the fused overview+roster
//! (`/view/swactor/actor-overview`) and the per-actor dossier
//! (`/view/swactor/actor-dossier`). Each is a self-contained `DashboardView`
//! instance ingesting the same channels.
use std::collections::{BTreeMap, VecDeque};
use std::time::{Duration, Instant};
use datastream::frame::{Frame, StreamId};
use parking_lot::RwLock;
use serde::Serialize;
use serde_json::{Value, json};
use crate::swactor::{RUNTIME_ACTORS, RUNTIME_STATS};
use crate::view::DashboardView;
use crate::{FrameEvent, StreamEvent};
const CHANNELS: &[&str] = &[RUNTIME_ACTORS, RUNTIME_STATS];
const HISTORY_CAP: usize = 512;
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(250);
const PER_ACTOR_HISTORY_CAP: usize = 120;
const GROWTH_WINDOW: usize = 12;
const LIVE_TTL: Duration = Duration::from_secs(8);
/// Read-only actor panel. One instance per served page so each page owns its
/// state independently; both ingest the same frames.
pub struct ActorPanelView {
id: &'static str,
title: &'static str,
path: &'static str,
html: &'static str,
state: RwLock<PanelState>,
}
impl ActorPanelView {
/// Fused overview + roster at `/view/swactor/actor-overview`.
pub fn overview() -> Self {
Self::new(
"swactor-actor-overview",
"Actor overview",
"swactor/actor-overview",
include_str!("actor_overview.html"),
)
}
/// Per-actor dossier at `/view/swactor/actor-dossier`.
pub fn dossier() -> Self {
Self::new(
"swactor-actor-dossier",
"Actor dossier",
"swactor/actor-dossier",
include_str!("actor_dossier.html"),
)
}
fn new(id: &'static str, title: &'static str, path: &'static str, html: &'static str) -> Self {
Self {
id,
title,
path,
html,
state: RwLock::new(PanelState::default()),
}
}
}
#[derive(Default)]
struct PanelState {
runtimes: BTreeMap<String, RuntimeState>,
}
struct RuntimeState {
stream: StreamEvent,
last_seen: Instant,
num_workers: Option<u32>,
uptime_ms: Option<u64>,
actors: BTreeMap<String, ActorState>,
history: VecDeque<HistorySample>,
}
impl RuntimeState {
fn new(stream: StreamEvent, now: Instant) -> Self {
Self {
stream,
last_seen: now,
num_workers: None,
uptime_ms: None,
actors: BTreeMap::new(),
history: VecDeque::with_capacity(HISTORY_CAP),
}
}
fn update(&mut self, channel: &str, payload: &[u8], now: Instant) {
self.last_seen = now;
let Ok(value) = serde_json::from_slice::<Value>(payload) else {
return;
};
match channel {
RUNTIME_ACTORS => self.apply_actors(&value, now),
RUNTIME_STATS => self.apply_stats(&value, now),
_ => {}
}
self.push_history(now);
}
fn apply_actors(&mut self, value: &Value, now: Instant) {
// Per-worker `worker_id` wrapper (DatastreamStatsHook shape) is the
// default placement for actors that do not carry one inline.
let wrapper_worker = u32_field(value, &["worker_id", "worker"]);
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
for actor in actors {
self.apply_actor(actor, now, wrapper_worker);
}
return;
}
// A bare actor object per frame (no envelope).
self.apply_actor(value, now, wrapper_worker);
}
fn apply_actor(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
return;
};
let actor = self
.actors
.entry(address.clone())
.or_insert_with(|| ActorState::new(address));
actor.apply_json(value, now, default_worker);
}
fn apply_stats(&mut self, value: &Value, now: Instant) {
if let Some(num_workers) = u32_field(value, &["num_workers", "workers_live"]) {
self.num_workers = Some(num_workers);
}
if let Some(uptime_ms) = u64_field(value, &["uptime_ms"]) {
self.uptime_ms = Some(uptime_ms);
}
// address -> worker_id mapping; runtime.stats `actors` is [[addr, wid]].
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
for entry in actors {
if let Some(items) = entry.as_array()
&& items.len() >= 2
&& let (Some(address), Some(worker_id)) =
(value_to_string(&items[0]), value_to_u32(&items[1]))
{
self.actors
.entry(address.clone())
.or_insert_with(|| ActorState::new(address))
.worker_id = Some(worker_id);
}
}
}
// Some publishers carry full per-actor detail under `actor_details`.
if let Some(details) = value.get("actor_details").and_then(Value::as_array) {
for actor in details {
self.apply_actor(actor, now, None);
}
}
}
fn push_history(&mut self, now: Instant) {
let totals = self.totals();
if let Some(last) = self.history.back_mut()
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
{
last.mailbox_depth = totals.mailbox_depth;
last.msg_per_sec = totals.msg_per_sec;
return;
}
if self.history.len() == HISTORY_CAP {
self.history.pop_front();
}
self.history.push_back(HistorySample {
at: now,
mailbox_depth: totals.mailbox_depth,
msg_per_sec: totals.msg_per_sec,
});
}
fn totals(&self) -> Totals {
let mut totals = Totals::default();
totals.actors = self.actors.len().min(u32::MAX as usize) as u32;
for actor in self.actors.values() {
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
totals.msg_per_sec += actor.msg_per_sec;
if actor.poisoned {
totals.poisoned = totals.poisoned.saturating_add(1);
}
}
totals
}
}
#[derive(Clone)]
struct ActorState {
address: String,
name: Option<String>,
actor_type: Option<String>,
message_type: Option<String>,
worker_id: Option<u32>,
mailbox_depth: u32,
mailbox_growth: f64,
messages_processed: u64,
msg_per_sec: f64,
last_msg_type: Option<String>,
poisoned: bool,
message_type_counts: Vec<(String, u64)>,
history: VecDeque<ActorHistorySample>,
last_update: Option<Instant>,
}
impl ActorState {
fn new(address: String) -> Self {
Self {
address,
name: None,
actor_type: None,
message_type: None,
worker_id: None,
mailbox_depth: 0,
mailbox_growth: 0.0,
messages_processed: 0,
msg_per_sec: 0.0,
last_msg_type: None,
poisoned: false,
message_type_counts: Vec::new(),
history: VecDeque::new(),
last_update: None,
}
}
fn apply_json(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
let elapsed = self
.last_update
.map(|then| now.duration_since(then).as_secs_f64())
.unwrap_or(0.0);
if let Some(name) = string_field(value, &["name"]).filter(|name| !name.is_empty()) {
self.name = Some(name);
}
if let Some(actor_type) = string_field(value, &["actor_type"]).filter(|t| !t.is_empty()) {
self.actor_type = Some(actor_type);
}
if let Some(message_type) = string_field(value, &["message_type"]).filter(|t| !t.is_empty()) {
self.message_type = Some(message_type);
}
if let Some(worker_id) = u32_field(value, &["worker_id", "worker"]) {
self.worker_id = Some(worker_id);
} else if self.worker_id.is_none() {
self.worker_id = default_worker;
}
assign_u32(&mut self.mailbox_depth, value, &["mailbox_depth", "queued"]);
assign_u64_rate(
&mut self.messages_processed,
&mut self.msg_per_sec,
value,
&["messages_processed", "processed", "messages_handled"],
elapsed,
);
if let Some(last) = string_field(
value,
&["last_msg_type", "last_message", "last_message_type"],
)
.filter(|last| !last.is_empty())
{
self.last_msg_type = Some(last);
}
if let Some(poisoned) = value.get("poisoned").and_then(Value::as_bool) {
self.poisoned = poisoned;
}
if let Some(counts) = parse_message_type_counts(value.get("message_type_counts")) {
self.message_type_counts = counts;
}
self.last_update = Some(now);
self.push_history(now);
self.recompute_growth();
}
fn push_history(&mut self, now: Instant) {
if let Some(last) = self.history.back_mut()
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
{
last.mailbox_depth = self.mailbox_depth;
last.msg_per_sec = self.msg_per_sec;
return;
}
if self.history.len() == PER_ACTOR_HISTORY_CAP {
self.history.pop_front();
}
self.history.push_back(ActorHistorySample {
at: now,
mailbox_depth: self.mailbox_depth,
msg_per_sec: self.msg_per_sec,
});
}
/// Mailbox depth/s over the recent window, for trend colouring.
fn recompute_growth(&mut self) {
let n = self.history.len();
if n < 2 {
self.mailbox_growth = 0.0;
return;
}
let start = n - n.min(GROWTH_WINDOW);
let first = &self.history[start];
let last = &self.history[n - 1];
let span = last.at.duration_since(first.at).as_secs_f64();
self.mailbox_growth = if span > 0.0 {
(last.mailbox_depth as f64 - first.mailbox_depth as f64) / span
} else {
0.0
};
}
}
#[derive(Clone, Copy)]
struct HistorySample {
at: Instant,
mailbox_depth: u32,
msg_per_sec: f64,
}
#[derive(Clone, Copy)]
struct ActorHistorySample {
at: Instant,
mailbox_depth: u32,
msg_per_sec: f64,
}
#[derive(Default)]
struct Totals {
actors: u32,
mailbox_depth: u32,
msg_per_sec: f64,
poisoned: u32,
}
#[derive(Serialize)]
struct PanelSnapshot {
runtimes: Vec<RuntimeSnapshot>,
}
#[derive(Serialize)]
struct RuntimeSnapshot {
stream: StreamSnapshot,
live: bool,
last_seen_ms_ago: u64,
summary: SummarySnapshot,
actors: Vec<ActorSnapshot>,
history: Vec<HistorySnapshot>,
}
#[derive(Serialize)]
struct StreamSnapshot {
key: String,
node: String,
life: u64,
}
#[derive(Serialize)]
struct SummarySnapshot {
actors: u32,
msg_per_sec: f64,
mailbox_depth: u32,
poisoned: u32,
uptime_ms: Option<u64>,
num_workers: Option<u32>,
}
#[derive(Serialize)]
struct ActorSnapshot {
address: String,
name: Option<String>,
actor_type: Option<String>,
message_type: Option<String>,
worker_id: Option<u32>,
/// Single derived display state. The frame carries only `poisoned`, so the
/// granularity is poisoned | running until the feed is enriched.
state: &'static str,
mailbox_depth: u32,
mailbox_growth: f64,
messages_processed: u64,
msg_per_sec: f64,
last_msg_type: Option<String>,
poisoned: bool,
message_type_counts: Vec<MessageTypeCountSnapshot>,
history: Vec<ActorHistorySnapshot>,
last_seen_ms_ago: u64,
}
#[derive(Serialize)]
struct MessageTypeCountSnapshot {
message_type: String,
count: u64,
}
#[derive(Serialize)]
struct HistorySnapshot {
ms_ago: u64,
msg_per_sec: f64,
mailbox_depth: u32,
}
#[derive(Serialize)]
struct ActorHistorySnapshot {
ms_ago: u64,
mailbox_depth: u32,
msg_per_sec: f64,
}
impl DashboardView for ActorPanelView {
fn id(&self) -> &'static str {
self.id
}
fn title(&self) -> &'static str {
self.title
}
fn path(&self) -> &'static str {
self.path
}
fn channels(&self) -> &'static [&'static str] {
CHANNELS
}
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
let now = Instant::now();
let mut state = self.state.write();
let key = stream_key(&event.stream);
state
.runtimes
.entry(key)
.or_insert_with(|| RuntimeState::new(event.stream.clone(), now))
.update(&event.channel, &event.payload, now);
}
fn snapshot_json(&self) -> Value {
let now = Instant::now();
let snapshot = PanelSnapshot {
runtimes: self
.state
.read()
.runtimes
.values()
.map(|runtime| runtime_snapshot(runtime, now))
.collect(),
};
serde_json::to_value(snapshot).unwrap_or_else(|_| json!({ "runtimes": [] }))
}
fn html(&self) -> Option<&'static str> {
Some(self.html)
}
}
fn runtime_snapshot(runtime: &RuntimeState, now: Instant) -> RuntimeSnapshot {
let totals = runtime.totals();
RuntimeSnapshot {
stream: StreamSnapshot {
key: stream_key(&runtime.stream),
node: runtime.stream.node.clone(),
life: runtime.stream.life,
},
live: now.duration_since(runtime.last_seen) <= LIVE_TTL,
last_seen_ms_ago: now.duration_since(runtime.last_seen).as_millis() as u64,
summary: SummarySnapshot {
actors: totals.actors,
msg_per_sec: totals.msg_per_sec,
mailbox_depth: totals.mailbox_depth,
poisoned: totals.poisoned,
uptime_ms: runtime.uptime_ms,
num_workers: runtime.num_workers,
},
actors: runtime
.actors
.values()
.map(|actor| ActorSnapshot {
address: actor.address.clone(),
name: actor.name.clone(),
actor_type: actor.actor_type.clone(),
message_type: actor.message_type.clone(),
worker_id: actor.worker_id,
state: if actor.poisoned { "poisoned" } else { "running" },
mailbox_depth: actor.mailbox_depth,
mailbox_growth: actor.mailbox_growth,
messages_processed: actor.messages_processed,
msg_per_sec: actor.msg_per_sec,
last_msg_type: actor.last_msg_type.clone(),
poisoned: actor.poisoned,
message_type_counts: actor
.message_type_counts
.iter()
.map(|(message_type, count)| MessageTypeCountSnapshot {
message_type: message_type.clone(),
count: *count,
})
.collect(),
history: actor
.history
.iter()
.map(|sample| ActorHistorySnapshot {
ms_ago: now.duration_since(sample.at).as_millis() as u64,
mailbox_depth: sample.mailbox_depth,
msg_per_sec: sample.msg_per_sec,
})
.collect(),
last_seen_ms_ago: actor
.last_update
.map(|then| now.duration_since(then).as_millis() as u64)
.unwrap_or(0),
})
.collect(),
history: runtime
.history
.iter()
.map(|sample| HistorySnapshot {
ms_ago: now.duration_since(sample.at).as_millis() as u64,
msg_per_sec: sample.msg_per_sec,
mailbox_depth: sample.mailbox_depth,
})
.collect(),
}
}
// --- tolerant JSON helpers (publisher-shape-agnostic readers) ---------------
fn assign_u32(slot: &mut u32, value: &Value, names: &[&str]) {
if let Some(v) = u32_field(value, names) {
*slot = v;
}
}
fn assign_u64_rate(slot: &mut u64, rate: &mut f64, value: &Value, names: &[&str], elapsed: f64) {
if let Some(next) = u64_field(value, names) {
if elapsed > 0.0 && next > *slot {
*rate = (next - *slot) as f64 / elapsed;
} else if next < *slot {
*rate = 0.0;
}
*slot = next;
}
}
fn u32_field(value: &Value, names: &[&str]) -> Option<u32> {
u64_field(value, names).and_then(|v| u32::try_from(v).ok())
}
fn u64_field(value: &Value, names: &[&str]) -> Option<u64> {
names
.iter()
.find_map(|name| value.get(*name).and_then(value_to_u64))
}
fn string_field(value: &Value, names: &[&str]) -> Option<String> {
names
.iter()
.find_map(|name| value.get(*name).and_then(value_to_string))
}
fn value_to_u32(value: &Value) -> Option<u32> {
value_to_u64(value).and_then(|v| u32::try_from(v).ok())
}
fn value_to_u64(value: &Value) -> Option<u64> {
value
.as_u64()
.or_else(|| value.as_str().and_then(|s| s.parse::<u64>().ok()))
}
fn value_to_string(value: &Value) -> Option<String> {
value.as_str().map(ToOwned::to_owned).or_else(|| {
if value.is_null() {
None
} else {
Some(value.to_string())
}
})
}
/// Accepts pair form `["Type", N]`, object form `{"ty": "Type", "count": N}`,
/// and map form `{"Type": N}`.
fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>> {
let value = value?;
if let Some(items) = value.as_array() {
let mut out = Vec::new();
for item in items {
if let Some(pair) = item.as_array()
&& pair.len() >= 2
&& let (Some(name), Some(count)) =
(value_to_string(&pair[0]), value_to_u64(&pair[1]))
{
out.push((name, count));
continue;
}
if let Some(name) = string_field(item, &["ty", "message_type", "type", "name"])
&& let Some(count) = u64_field(item, &["count"])
{
out.push((name, count));
}
}
return Some(out);
}
if let Some(map) = value.as_object() {
let mut out: Vec<(String, u64)> = map
.iter()
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
.collect();
out.sort_by(|a, b| b.1.cmp(&a.1));
return Some(out);
}
None
}
fn stream_key(stream: &StreamEvent) -> String {
format!("{}#{}", stream.node, stream.life)
}

View file

@ -2,10 +2,12 @@ use std::sync::Arc;
use crate::view::DashboardView; use crate::view::DashboardView;
mod actor_view;
mod worker_page; mod worker_page;
mod worker_view; mod worker_view;
pub use worker_view::SwactorWorkerView; pub use worker_view::SwactorWorkerView;
pub use actor_view::ActorPanelView;
pub const RUNTIME_STATS: &str = "runtime.stats"; pub const RUNTIME_STATS: &str = "runtime.stats";
pub const RUNTIME_WORKERS: &str = "runtime.workers"; pub const RUNTIME_WORKERS: &str = "runtime.workers";
@ -14,3 +16,13 @@ pub const RUNTIME_ACTORS: &str = "runtime.actors";
pub fn worker_view() -> Arc<dyn DashboardView> { pub fn worker_view() -> Arc<dyn DashboardView> {
Arc::new(SwactorWorkerView::default()) Arc::new(SwactorWorkerView::default())
} }
/// Built-in actor overview + roster view (`/view/swactor/actor-overview`).
pub fn actor_overview_view() -> Arc<dyn DashboardView> {
Arc::new(ActorPanelView::overview())
}
/// Built-in actor dossier view (`/view/swactor/actor-dossier`).
pub fn actor_dossier_view() -> Arc<dyn DashboardView> {
Arc::new(ActorPanelView::dossier())
}

View file

@ -758,9 +758,11 @@ impl<'a> RuntimeActorStatsRecord<'a> {
actors: snapshots actors: snapshots
.iter() .iter()
.map(|snapshot| RuntimeActorSnapshotRecord { .map(|snapshot| RuntimeActorSnapshotRecord {
address: snapshot.address.to_string(), address: snapshot.address.to_full_hex(),
mailbox_depth: snapshot.mailbox_depth, mailbox_depth: snapshot.mailbox_depth,
last_msg_type: snapshot.last_msg_type, last_msg_type: snapshot.last_msg_type,
actor_type: snapshot.actor_type,
message_type: snapshot.message_type,
messages_processed: snapshot.messages_processed, messages_processed: snapshot.messages_processed,
poisoned: snapshot.poisoned, poisoned: snapshot.poisoned,
message_type_counts: snapshot message_type_counts: snapshot
@ -779,6 +781,8 @@ struct RuntimeActorSnapshotRecord<'a> {
address: String, address: String,
mailbox_depth: usize, mailbox_depth: usize,
last_msg_type: Option<&'static str>, last_msg_type: Option<&'static str>,
actor_type: Option<&'static str>,
message_type: Option<&'static str>,
messages_processed: u64, messages_processed: u64,
poisoned: bool, poisoned: bool,
message_type_counts: Vec<RuntimeMessageTypeCount<'a>>, message_type_counts: Vec<RuntimeMessageTypeCount<'a>>,

View file

@ -300,6 +300,8 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
address: actor, address: actor,
mailbox_depth: 3, mailbox_depth: 3,
last_msg_type: Some("Ping"), last_msg_type: Some("Ping"),
actor_type: Some("TestActor"),
message_type: Some("Ping"),
messages_processed: 5, messages_processed: 5,
poisoned: false, poisoned: false,
message_type_counts: vec![("Ping", 5)], message_type_counts: vec![("Ping", 5)],
@ -313,11 +315,13 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
assert_eq!(delivery.channel.channel, runtime); assert_eq!(delivery.channel.channel, runtime);
let json: Value = serde_json::from_slice(&delivery.payload).unwrap(); let json: Value = serde_json::from_slice(&delivery.payload).unwrap();
assert_eq!(json["worker_id"], 2); assert_eq!(json["worker_id"], 2);
assert_eq!(json["actors"][0]["address"], actor.to_string()); assert_eq!(json["actors"][0]["address"], actor.to_full_hex());
assert_eq!(json["actors"][0]["mailbox_depth"], 3); assert_eq!(json["actors"][0]["mailbox_depth"], 3);
assert_eq!(json["actors"][0]["last_msg_type"], "Ping"); assert_eq!(json["actors"][0]["last_msg_type"], "Ping");
assert_eq!(json["actors"][0]["messages_processed"], 5); assert_eq!(json["actors"][0]["messages_processed"], 5);
assert_eq!(json["actors"][0]["message_type_counts"][0]["ty"], "Ping"); assert_eq!(json["actors"][0]["message_type_counts"][0]["ty"], "Ping");
assert_eq!(json["actors"][0]["actor_type"], "TestActor");
assert_eq!(json["actors"][0]["message_type"], "Ping");
} }
fn positions(events: &[DatastreamEvent]) -> Vec<u64> { fn positions(events: &[DatastreamEvent]) -> Vec<u64> {

View file

@ -19,9 +19,8 @@ iroh = "0.98"
# `test-utils` exposes `CaRootsConfig::insecure_skip_verify()` so clients can # `test-utils` exposes `CaRootsConfig::insecure_skip_verify()` so clients can
# trust operator-controlled custom relays with self-signed QAD certs. # trust operator-controlled custom relays with self-signed QAD certs.
iroh-relay = { version = "0.98", features = ["test-utils"] } iroh-relay = { version = "0.98", features = ["test-utils"] }
tokio = { version = "1", features = ["rt-multi-thread", "time"] } tokio.workspace = true
parking_lot = "0.12" parking_lot = "0.12"
[dev-dependencies] [dev-dependencies]
iroh-relay = { version = "0.98", features = ["server", "test-utils"] } iroh-relay = { version = "0.98", features = ["server", "test-utils"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "io-util", "time"] }

2
rust-toolchain.toml Normal file
View file

@ -0,0 +1,2 @@
[toolchain]
channel = "nightly-2026-02-07"

View file

@ -162,6 +162,18 @@ impl ActorAddress {
crate::get_random(&mut bytes); crate::get_random(&mut bytes);
Self(bytes) Self(bytes)
} }
/// Full 64-character hex encoding of all 32 bytes, for displays that need
/// the untruncated address (dashboards). [`Display`](std::fmt::Display)
/// stays short for logs.
pub fn to_full_hex(&self) -> String {
use std::fmt::Write;
let mut s = String::with_capacity(64);
for b in &self.0 {
let _ = write!(s, "{:02x}", b);
}
s
}
} }
// ─── Environment ───────────────────────────────────────────────────────────── // ─── Environment ─────────────────────────────────────────────────────────────

View file

@ -124,6 +124,8 @@ pub struct ActorSnapshot {
pub address: ActorAddress, pub address: ActorAddress,
pub mailbox_depth: usize, pub mailbox_depth: usize,
pub last_msg_type: Option<&'static str>, pub last_msg_type: Option<&'static str>,
pub actor_type: Option<&'static str>,
pub message_type: Option<&'static str>,
pub messages_processed: u64, pub messages_processed: u64,
pub poisoned: bool, pub poisoned: bool,
/// Per-message-type counts, sorted descending by count. /// Per-message-type counts, sorted descending by count.

View file

@ -998,10 +998,13 @@ impl ActorPool {
let mut type_counts: Vec<(&'static str, u64)> = let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect(); slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1)); type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let metadata = slot.actor.metadata();
ActorSnapshot { ActorSnapshot {
address: addr, address: addr,
mailbox_depth: slot.mailbox.len(), mailbox_depth: slot.mailbox.len(),
last_msg_type: slot.last_msg_type, last_msg_type: slot.last_msg_type,
actor_type: Some(metadata.actor_type_name),
message_type: Some(metadata.message_type_name),
messages_processed: slot.messages_processed, messages_processed: slot.messages_processed,
poisoned: slot.poisoned, poisoned: slot.poisoned,
message_type_counts: type_counts, message_type_counts: type_counts,

View file

@ -8,7 +8,7 @@ publish = false
reqwest = { version = "0.12", features = ["json"] } reqwest = { version = "0.12", features = ["json"] }
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
tokio = { version = "1", features = ["macros", "rt", "time"] } tokio.workspace = true
urlencoding = "2" urlencoding = "2"
[dev-dependencies] [dev-dependencies]

View file

@ -280,7 +280,7 @@ OPTIONS:
Select the runtime provider Select the runtime provider
--config <path> Load config overlay --config <path> Load config overlay
--pipeline-stages <count> Number of pipeline stages --pipeline-stages <count> Number of pipeline stages
--cached-model[=<path>] Use discovered or explicit cached GGUF model --cached-model[=<path>] Use discovered or explicit cached GGUF model (default for --process)
--dump-logs[=<path>] Write datastream frame log --dump-logs[=<path>] Write datastream frame log
--run-id <id> Override run id --run-id <id> Override run id
--skip-rebuild Reuse existing Cargo artifacts --skip-rebuild Reuse existing Cargo artifacts