feat: actor view panels for the dashboard
Add read-only actor overview and per-actor dossier pages to the dashboard, fed by enriched per-actor runtime snapshots. - `dashboard/swactor/actor_view`: new `ActorPanelView`, a tolerant frame consumer over `runtime.actors`/`runtime.stats` that folds per-actor snapshots and serves `/view/swactor/actor-overview` (roster) and `/view/swactor/actor-dossier` (per-actor detail), each backed by an embedded HTML template (`actor_overview.html`, `actor_dossier.html`) - `dashboard`: register both views in `DashboardHandle` and export `actor_overview_view()`/`actor_dossier_view()` from the swactor module - `swactor` core: enrich `ActorSnapshot` with `actor_type` and `message_type` (populated from `slot.actor.metadata()` in `ActorPool`) and add `ActorAddress::to_full_hex()` for untruncated display - `myelin/orchestration`: publish actor stats to the dashboard via a `runtime.actors` channel producer (`stats_hook_on`) threaded through the distribution stack, and carry the orchestrator actor address into readiness signaling - workspace `Cargo.toml`: add `default-members` for native iteration and a centralized `[workspace.dependencies] tokio` so members share one feature set; `dashboard/Cargo.toml` switches to `tokio.workspace = true` Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
parent
ef9e1c98a3
commit
2bf11e0fdb
35 changed files with 1490 additions and 140 deletions
20
Cargo.toml
20
Cargo.toml
|
|
@ -15,7 +15,27 @@ members = [
|
|||
"xtask",
|
||||
"tools/vastai",
|
||||
]
|
||||
default-members = [
|
||||
".",
|
||||
"crates/process",
|
||||
"crates/provisioning",
|
||||
"crates/transport",
|
||||
"crates/distribution",
|
||||
"crates/iroh-driver",
|
||||
"crates/datastream",
|
||||
"crates/data-plane",
|
||||
"crates/dashboard",
|
||||
"apps/myelin",
|
||||
"tools/vastai",
|
||||
]
|
||||
# The language bindings (python, wasm) and xtask are built on demand
|
||||
# (-p / --workspace / cargo-xtask), not during normal native iteration.
|
||||
exclude = ["crates/bindings/wasm-crypto", "examples"]
|
||||
[workspace.dependencies]
|
||||
# Centralized so every member resolves tokio with the SAME feature set.
|
||||
# Without this, members declared tokio with different features, so building
|
||||
# one member vs another (or `run` vs `test`) recompiled tokio each time.
|
||||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
|
||||
|
||||
[package]
|
||||
name = "swactor"
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ swactor-process = { path = "../../crates/process" }
|
|||
distribution = { path = "../../crates/distribution" }
|
||||
iroh-driver = { path = "../../crates/iroh-driver" }
|
||||
iroh = "0.98"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
|
||||
tokio.workspace = true
|
||||
swactor-vastai = { path = "../../tools/vastai" }
|
||||
parking_lot = "0.12"
|
||||
blake3 = "1"
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ OPTIONS:
|
|||
--relay-mode <mode> Relay mode: default or disabled
|
||||
--relay-url <url> Custom relay URL passed to myelin-orchestrator
|
||||
--endpoint-addr-mask <mask> Endpoint address mask: full or relay-only
|
||||
--cached-model[=<path>] Use discovered or explicit cached GGUF model
|
||||
--cached-model[=<path>] Use discovered or explicit cached GGUF model (default for --process)
|
||||
--dump-logs[=<path>] Write datastream frame log
|
||||
--run-id <id> Override run id
|
||||
--skip-rebuild Reuse existing Cargo artifacts
|
||||
|
|
@ -793,9 +793,15 @@ impl Config {
|
|||
let gpu_run = args.gpu || env_flag(MYELIN_CHAT_GPU_RUN_ENV, false);
|
||||
let endpoint_addr_mask = Self::endpoint_addr_mask(&args, &toml)?;
|
||||
let (relay_mode, relay_url) = Self::relay_settings(&args, &toml, endpoint_addr_mask)?;
|
||||
let cached_model = Self::cached_model_source(&args, gpu_run, &provider)
|
||||
.map(CachedModelConfig::from_source)
|
||||
.transpose()?;
|
||||
let cached_model = match Self::cached_model_source(&args, &provider) {
|
||||
None => None,
|
||||
Some(CachedModelSource::Path(path)) => Some(CachedModelConfig::from_path(path)?),
|
||||
Some(CachedModelSource::Discover) => match CachedModelConfig::discover() {
|
||||
Ok(config) => Some(config),
|
||||
Err(error) if args.cached_model.is_some() => return Err(error),
|
||||
Err(_) => None,
|
||||
},
|
||||
};
|
||||
let model = Self::model_config(&provider, &toml, cached_model.as_ref())?;
|
||||
let datastream_frame_log = Self::datastream_frame_log(&args, &toml);
|
||||
let vastai = if provider == provider_kind::vastai() {
|
||||
|
|
@ -876,16 +882,15 @@ impl Config {
|
|||
Ok((relay_mode, relay_url))
|
||||
}
|
||||
|
||||
fn cached_model_source(
|
||||
args: &ParsedArgs,
|
||||
gpu_run: bool,
|
||||
provider: &ProviderKind,
|
||||
) -> Option<CachedModelSource> {
|
||||
/// Resolve the cached-model source. The process provider defaults to
|
||||
/// best-effort discovery of `.model-cache/` so `cargo myelin-chat` runs a
|
||||
/// cached GGUF model without explicit flags. Explicit `--cached-model` is
|
||||
/// always honored (and stays strict); the default degrades gracefully to
|
||||
/// the normal download path when no cached model is present.
|
||||
fn cached_model_source(args: &ParsedArgs, provider: &ProviderKind) -> Option<CachedModelSource> {
|
||||
match &args.cached_model {
|
||||
Some(source) => Some(source.clone()),
|
||||
None if gpu_run && provider == &provider_kind::process() => {
|
||||
Some(CachedModelSource::Discover)
|
||||
}
|
||||
None if provider == &provider_kind::process() => Some(CachedModelSource::Discover),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
|
@ -2122,13 +2127,6 @@ struct CachedModelConfig {
|
|||
}
|
||||
|
||||
impl CachedModelConfig {
|
||||
fn from_source(source: CachedModelSource) -> Result<Self, String> {
|
||||
match source {
|
||||
CachedModelSource::Discover => Self::discover(),
|
||||
CachedModelSource::Path(path) => Self::from_path(path),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_path(path: PathBuf) -> Result<Self, String> {
|
||||
let metadata = fs::metadata(&path)
|
||||
.map_err(|e| format!("stat cached model {}: {e}", path.display()))?;
|
||||
|
|
@ -2517,6 +2515,34 @@ tag = " alias "
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_defaults_use_cached_model_for_process_when_present() {
|
||||
let temp = TempDir::new("cached-model-default");
|
||||
|
||||
with_process_state(&[], Some(temp.path()), || {
|
||||
let cache_dir = temp.path().join(REPO_MODEL_CACHE_DIR);
|
||||
fs::create_dir_all(&cache_dir).expect("create model cache dir");
|
||||
fs::write(
|
||||
cache_dir.join(DEFAULT_PIPELINE_CACHED_MODEL_FILE),
|
||||
Vec::<u8>::new(),
|
||||
)
|
||||
.expect("seed cached model file");
|
||||
|
||||
let defaults = Config::from_args(Vec::<String>::new()).expect("defaults resolve");
|
||||
|
||||
assert_eq!(defaults.provider, provider_kind::process());
|
||||
let cached_model = defaults
|
||||
.cached_model
|
||||
.expect("process provider discovers a cached model by default");
|
||||
assert!(
|
||||
cached_model
|
||||
.host_path
|
||||
.ends_with(DEFAULT_PIPELINE_CACHED_MODEL_FILE),
|
||||
"discovered the seeded cached model"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_max_tokens_drives_orchestrator_args_and_submit_prompt() {
|
||||
let temp = TempDir::new("config-max-tokens");
|
||||
|
|
|
|||
|
|
@ -170,6 +170,19 @@ pub(crate) enum NodeAgentMsg {
|
|||
WorkerCrashed {
|
||||
reason: Option<String>,
|
||||
},
|
||||
StepFailed {
|
||||
step_id: u64,
|
||||
},
|
||||
ObjectFailed {
|
||||
edge_id: u64,
|
||||
object_id: Option<u64>,
|
||||
},
|
||||
OutputFault {
|
||||
edge_id: u64,
|
||||
},
|
||||
EdgeFault {
|
||||
edge_id: u64,
|
||||
},
|
||||
StopRun {
|
||||
run_id: u64,
|
||||
},
|
||||
|
|
@ -236,9 +249,6 @@ pub(crate) enum StageCommandWire {
|
|||
layer_end_exclusive: u32,
|
||||
stage_shard_plan: Option<StageShardPlan>,
|
||||
},
|
||||
RewireEdge {
|
||||
edge_id: u64,
|
||||
},
|
||||
ExecuteStep {
|
||||
step_id: u64,
|
||||
input_edge_id: u64,
|
||||
|
|
@ -531,6 +541,27 @@ impl NodeAgentActor {
|
|||
self.last_worker_crash = reason;
|
||||
self.core.observe(stage::StageEvent::WorkerCrashed)
|
||||
}
|
||||
NodeAgentMsg::StepFailed { step_id } => {
|
||||
self.core.observe(stage::StageEvent::StepFailed {
|
||||
step_id: stage::StepId(step_id),
|
||||
})
|
||||
}
|
||||
NodeAgentMsg::ObjectFailed { edge_id, object_id } => {
|
||||
self.core.observe(stage::StageEvent::ObjectFailed {
|
||||
edge_id: stage::EdgeId(edge_id),
|
||||
object_id: object_id.map(stage::ObjectId),
|
||||
})
|
||||
}
|
||||
NodeAgentMsg::OutputFault { edge_id } => {
|
||||
self.core.observe(stage::StageEvent::OutputFault {
|
||||
edge_id: stage::EdgeId(edge_id),
|
||||
})
|
||||
}
|
||||
NodeAgentMsg::EdgeFault { edge_id } => {
|
||||
self.core.observe(stage::StageEvent::EdgeFault {
|
||||
edge_id: stage::EdgeId(edge_id),
|
||||
})
|
||||
}
|
||||
NodeAgentMsg::StopRun { run_id } => self.core.observe(stage::StageEvent::StopRun {
|
||||
run_id: stage::RunId(run_id),
|
||||
}),
|
||||
|
|
@ -692,7 +723,6 @@ impl From<&stage::StageCommand> for StageCommandWire {
|
|||
layer_end_exclusive: range.end_exclusive,
|
||||
stage_shard_plan: shard_plan.clone(),
|
||||
},
|
||||
stage::StageCommand::RewireEdge { edge_id } => Self::RewireEdge { edge_id: edge_id.0 },
|
||||
stage::StageCommand::ExecuteStep(step) => Self::ExecuteStep {
|
||||
step_id: step.step_id.0,
|
||||
input_edge_id: step.input.edge_id.0,
|
||||
|
|
|
|||
|
|
@ -1013,7 +1013,7 @@ impl WorkerEdgeRuntime {
|
|||
);
|
||||
let step_started = Instant::now();
|
||||
let mut pump = || pump_network(driver, stack);
|
||||
let committed_bytes = worker.execute_step(
|
||||
let committed_bytes = match worker.execute_step(
|
||||
u64::from(config.stage_index) + 1,
|
||||
step_id,
|
||||
object_id,
|
||||
|
|
@ -1027,7 +1027,15 @@ impl WorkerEdgeRuntime {
|
|||
config,
|
||||
datastream,
|
||||
&mut pump,
|
||||
)?;
|
||||
) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
let _ = stack
|
||||
.runtime
|
||||
.send_to(node_actor, NodeAgentMsg::StepFailed { step_id });
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
let helper_execute_ms = duration_ms_u64(step_started.elapsed());
|
||||
let egress_read_started = Instant::now();
|
||||
let record = {
|
||||
|
|
@ -1035,9 +1043,16 @@ impl WorkerEdgeRuntime {
|
|||
let lease = arena
|
||||
.lookup_lease(arena::RingId(output_ring_id))
|
||||
.ok_or_else(|| format!("outbound ring {output_ring_id} lease missing"))?;
|
||||
arena
|
||||
.read_arena(lease.layout.data_offset, committed_bytes)
|
||||
.map_err(|e| format!("read egress ring: {e}"))?
|
||||
arena.read_arena(lease.layout.data_offset, committed_bytes)
|
||||
};
|
||||
let record = match record {
|
||||
Ok(record) => record,
|
||||
Err(e) => {
|
||||
let _ = stack
|
||||
.runtime
|
||||
.send_to(node_actor, NodeAgentMsg::OutputFault { edge_id: outbound.edge_id });
|
||||
return Err(format!("read egress ring: {e}"));
|
||||
}
|
||||
};
|
||||
let egress_read_ms = duration_ms_u64(egress_read_started.elapsed());
|
||||
let record_bytes = record.len();
|
||||
|
|
@ -1067,7 +1082,12 @@ impl WorkerEdgeRuntime {
|
|||
.as_ref()
|
||||
.ok_or_else(|| "outbound edge sender missing".to_owned())?;
|
||||
let edge_send_started = Instant::now();
|
||||
sender.send(record)?;
|
||||
if let Err(e) = sender.send(record) {
|
||||
let _ = stack
|
||||
.runtime
|
||||
.send_to(node_actor, NodeAgentMsg::OutputFault { edge_id: outbound.edge_id });
|
||||
return Err(e);
|
||||
}
|
||||
let edge_send_ms = duration_ms_u64(edge_send_started.elapsed());
|
||||
node_stage(
|
||||
datastream,
|
||||
|
|
@ -1131,8 +1151,17 @@ impl WorkerEdgeRuntime {
|
|||
buffer.extend_from_slice(&bytes);
|
||||
let buffered_bytes = buffer.len();
|
||||
let mut records = Vec::new();
|
||||
while let Some(record) = take_complete_ingress_record(buffer, inbound.object_spec)? {
|
||||
records.push(record);
|
||||
loop {
|
||||
match take_complete_ingress_record(buffer, inbound.object_spec) {
|
||||
Ok(Some(record)) => records.push(record),
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
let _ = stack
|
||||
.runtime
|
||||
.send_to(node_actor, NodeAgentMsg::ObjectFailed { edge_id, object_id: None });
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
(records, buffered_bytes)
|
||||
};
|
||||
|
|
@ -1171,14 +1200,22 @@ impl WorkerEdgeRuntime {
|
|||
}),
|
||||
);
|
||||
let object_load_started = Instant::now();
|
||||
let loaded = worker.ring_readable(
|
||||
let loaded = match worker.ring_readable(
|
||||
ring_id,
|
||||
edge_id,
|
||||
inbound.object_spec,
|
||||
config,
|
||||
datastream,
|
||||
&mut || {},
|
||||
)?;
|
||||
) {
|
||||
Ok(loaded) => loaded,
|
||||
Err(e) => {
|
||||
let _ = stack
|
||||
.runtime
|
||||
.send_to(node_actor, NodeAgentMsg::ObjectFailed { edge_id, object_id: Some(record.object_id) });
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
let object_load_ms = duration_ms_u64(object_load_started.elapsed());
|
||||
let key = ObjectKey {
|
||||
edge_id,
|
||||
|
|
@ -1510,11 +1547,9 @@ impl WorkerEdgeRuntime {
|
|||
.runtime
|
||||
.send_to(
|
||||
node_actor,
|
||||
NodeAgentMsg::WorkerCrashed {
|
||||
reason: Some(format!("edge {} faulted: {reason:?}", edge_id.0)),
|
||||
},
|
||||
NodeAgentMsg::EdgeFault { edge_id: edge_id.0 },
|
||||
)
|
||||
.map_err(|e| format!("mark worker crashed after edge fault: {e}"))?;
|
||||
.map_err(|e| format!("report edge fault: {e}"))?;
|
||||
return Err(format!("edge {} faulted: {reason:?}", edge_id.0));
|
||||
}
|
||||
edge::EdgeLifecycleEvent::EdgeStopped { .. } => {}
|
||||
|
|
@ -1691,6 +1726,8 @@ fn run() -> Result<(), String> {
|
|||
)?;
|
||||
}
|
||||
|
||||
let mut datastream = NodeDatastream::new(&config);
|
||||
let worker_stats_hook = datastream.producer.stats_hook();
|
||||
let stack = DistributionRuntimeStack::new_with_codecs(
|
||||
driver.node_id(),
|
||||
DistributedNodeConfig::default(),
|
||||
|
|
@ -1698,6 +1735,7 @@ fn run() -> Result<(), String> {
|
|||
register_myelin_actor_codecs(registry);
|
||||
datastream::wire::register_datastream_codec(registry);
|
||||
},
|
||||
Some(worker_stats_hook),
|
||||
);
|
||||
boot(
|
||||
"distribution_stack",
|
||||
|
|
@ -1750,7 +1788,6 @@ fn run() -> Result<(), String> {
|
|||
};
|
||||
let arena_fd = arena_manager.lock().arena_fd();
|
||||
|
||||
let mut datastream = NodeDatastream::new(&config);
|
||||
let node_boot = |ds: &mut NodeDatastream, phase: &str, status: &str, detail: Value| {
|
||||
emit_node_event(ds, &config, NODE_BOOTSTRAP_CHANNEL, phase, status, detail)
|
||||
};
|
||||
|
|
@ -3560,15 +3597,6 @@ fn handle_stage_command(
|
|||
);
|
||||
Ok(())
|
||||
}
|
||||
StageCommandWire::RewireEdge { .. } => {
|
||||
node_stage(
|
||||
datastream,
|
||||
"rewire_edge",
|
||||
"skipped",
|
||||
json!({"reason":"not implemented in myelin-worker image path"}),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
StageCommandWire::ReleaseInputHandle { handle_id, .. } => {
|
||||
edge_runtime.release_input_handle(handle_id, worker, config, datastream, driver, stack)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,6 +68,13 @@ pub(crate) enum OrchestratorMsg {
|
|||
stage_index: u32,
|
||||
},
|
||||
ObserveTokenEndpointsStopped,
|
||||
ObserveOperatorStop {
|
||||
run_id: u64,
|
||||
},
|
||||
ObserveMembershipLost {
|
||||
run_id: u64,
|
||||
node_id: u64,
|
||||
},
|
||||
AdvanceTimeMs(u64),
|
||||
Snapshot {
|
||||
reply_to: ActorAddress,
|
||||
|
|
@ -131,7 +138,6 @@ pub(crate) enum RunCommandWire {
|
|||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) enum LifecycleEventWire {
|
||||
RunRejected { run_id: u64 },
|
||||
RunFaulted { run_id: u64 },
|
||||
RunCompleted { run_id: u64 },
|
||||
RunOperatorStopped { run_id: u64 },
|
||||
|
|
@ -272,6 +278,17 @@ impl OrchestratorActor {
|
|||
OrchestratorMsg::ObserveTokenEndpointsStopped => {
|
||||
self.core.observe(core::RunEvent::TokenEndpointsStopped)
|
||||
}
|
||||
OrchestratorMsg::ObserveOperatorStop { run_id } => {
|
||||
self.core.observe(core::RunEvent::OperatorStop {
|
||||
run_id: core::RunId(run_id),
|
||||
});
|
||||
}
|
||||
OrchestratorMsg::ObserveMembershipLost { run_id, node_id } => {
|
||||
self.core.observe(core::RunEvent::MembershipLost {
|
||||
run_id: core::RunId(run_id),
|
||||
node_id: core::NodeId(node_id),
|
||||
});
|
||||
}
|
||||
OrchestratorMsg::AdvanceTimeMs(delta) => self.core.advance_time_ms(delta),
|
||||
}
|
||||
}
|
||||
|
|
@ -449,9 +466,6 @@ impl From<&core::RunCommand> for RunCommandWire {
|
|||
impl From<&core::LifecycleEvent> for LifecycleEventWire {
|
||||
fn from(event: &core::LifecycleEvent) -> Self {
|
||||
match event {
|
||||
core::LifecycleEvent::RunRejected { run_id, .. } => {
|
||||
Self::RunRejected { run_id: run_id.0 }
|
||||
}
|
||||
core::LifecycleEvent::RunFaulted { run_id, .. } => {
|
||||
Self::RunFaulted { run_id: run_id.0 }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ use crate::node_actor::{
|
|||
#[cfg(feature = "dashboard")]
|
||||
use crate::observability::dashboard_view::MyelinClusterDashboardView;
|
||||
use crate::observability::{benchmark, frame_archive::FrameArchive};
|
||||
use crate::orchestration::actor::{OrchestratorActor, OrchestratorReport};
|
||||
use crate::orchestration::actor::{OrchestratorActor, OrchestratorMsg, OrchestratorReport};
|
||||
use crate::orchestration::config::{DEFAULT_CONFIG_PATH, TomlConfigOverlay};
|
||||
const PROVIDER_START_MAX_ATTEMPTS: usize = 4;
|
||||
|
||||
|
|
@ -56,6 +56,7 @@ use datastream::{
|
|||
};
|
||||
use distribution::node::DistributedNodeConfig;
|
||||
use distribution::telemetry::{MembershipTransition, SwimProbeEvent};
|
||||
use distribution::swim::telemetry::ObservedTransition;
|
||||
use distribution::types::{MemberState, NodeId as DistNodeId};
|
||||
use iroh::EndpointAddr;
|
||||
use iroh_driver::{
|
||||
|
|
@ -283,6 +284,8 @@ where
|
|||
"connectivity_preflight":"ready",
|
||||
}),
|
||||
);
|
||||
let actors_channel = orch_datastream.channel_by_name("runtime.actors");
|
||||
let orch_stats_hook = orch_datastream.producer.stats_hook_on(actors_channel);
|
||||
let stack = DistributionRuntimeStack::new_with_codecs(
|
||||
driver.node_id(),
|
||||
DistributedNodeConfig::default(),
|
||||
|
|
@ -290,6 +293,7 @@ where
|
|||
register_myelin_actor_codecs(registry);
|
||||
datastream::wire::register_datastream_codec(registry);
|
||||
},
|
||||
Some(orch_stats_hook),
|
||||
);
|
||||
bootstrap(
|
||||
&mut orch_datastream,
|
||||
|
|
@ -455,7 +459,7 @@ where
|
|||
tx: Mutex::new(obs_tx),
|
||||
}));
|
||||
let pipeline_coordinator_endpoint = coordinator_endpoint.clone();
|
||||
let (mut provisioned_nodes, ready) = start_and_provision_workers(
|
||||
let (mut provisioned_nodes, ready, swim_to_node) = start_and_provision_workers(
|
||||
provisioner,
|
||||
&config,
|
||||
pipeline_plan.as_ref(),
|
||||
|
|
@ -473,6 +477,7 @@ where
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
},
|
||||
sink,
|
||||
coordinator_endpoint,
|
||||
|
|
@ -547,6 +552,7 @@ where
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
},
|
||||
&work_rx,
|
||||
&prompt_events,
|
||||
|
|
@ -558,6 +564,7 @@ where
|
|||
tokenizer_reply_actor,
|
||||
pipeline_plan.as_ref(),
|
||||
ready.first_stage.endpoint.clone(),
|
||||
&swim_to_node,
|
||||
);
|
||||
if let Err(error) = &result {
|
||||
bootstrap(
|
||||
|
|
@ -2024,6 +2031,7 @@ struct RuntimeReadyAckLoop<'a> {
|
|||
run_id: u64,
|
||||
orchestrator_node_id: u64,
|
||||
provider: &'a ProviderKind,
|
||||
orchestrator_actor: ActorAddress,
|
||||
}
|
||||
|
||||
fn wait_for_runtime_ready_acks(
|
||||
|
|
@ -2045,6 +2053,7 @@ fn wait_for_runtime_ready_acks(
|
|||
run_id,
|
||||
orchestrator_node_id,
|
||||
provider,
|
||||
..
|
||||
} = ctx;
|
||||
let bootstrap = |ds: &mut OrchDatastream, phase: &str, status: &str, detail: Value| {
|
||||
ds.emit_bootstrap(
|
||||
|
|
@ -2232,7 +2241,7 @@ fn start_and_provision_workers(
|
|||
coordinator: EndpointAddr,
|
||||
pipeline_coordinator: EndpointAddr,
|
||||
orchestrator_actor: ActorAddress,
|
||||
) -> Result<(ProvisionedClusterGuard, PromptRuntimeReady), String> {
|
||||
) -> Result<(ProvisionedClusterGuard, PromptRuntimeReady, BTreeMap<DistNodeId, u64>), String> {
|
||||
let RuntimeReadyAckLoop {
|
||||
driver,
|
||||
stack,
|
||||
|
|
@ -2439,6 +2448,7 @@ fn start_and_provision_workers(
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
},
|
||||
&expected_node_ids,
|
||||
) {
|
||||
|
|
@ -2468,6 +2478,7 @@ fn start_and_provision_workers(
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
}) {
|
||||
Ok(ready) => ready,
|
||||
Err(error) => {
|
||||
|
|
@ -2512,6 +2523,7 @@ fn start_and_provision_workers(
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
},
|
||||
&ack_targets,
|
||||
&pipeline_coordinator,
|
||||
|
|
@ -2555,6 +2567,7 @@ fn start_and_provision_workers(
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
},
|
||||
expected_node_ids.len(),
|
||||
pipeline_plan.expect("pipeline mode requires plan"),
|
||||
|
|
@ -2578,6 +2591,7 @@ fn start_and_provision_workers(
|
|||
run_id: config.run_id,
|
||||
orchestrator_node_id: config.node_id,
|
||||
provider: &config.provider,
|
||||
orchestrator_actor,
|
||||
},
|
||||
config.stage_index,
|
||||
)
|
||||
|
|
@ -2640,7 +2654,11 @@ fn start_and_provision_workers(
|
|||
final_stage: ready,
|
||||
}
|
||||
};
|
||||
Ok((provisioned_nodes, prompt_ready))
|
||||
let swim_to_node = readies
|
||||
.iter()
|
||||
.map(|(node_id, ready)| (ready.swim_node_id, *node_id))
|
||||
.collect::<BTreeMap<DistNodeId, u64>>();
|
||||
Ok((provisioned_nodes, prompt_ready, swim_to_node))
|
||||
}
|
||||
|
||||
fn stage_node_specs(
|
||||
|
|
@ -2999,6 +3017,7 @@ fn wait_for_weights_loaded_count(
|
|||
run_id,
|
||||
orchestrator_node_id: node_id,
|
||||
provider,
|
||||
..
|
||||
} = ctx;
|
||||
let expected_stages = pipeline_plan
|
||||
.stages
|
||||
|
|
@ -4126,6 +4145,7 @@ fn wait_for_runtime_ready(ctx: RuntimeReadyAckLoop<'_>) -> Result<RuntimeReady,
|
|||
run_id,
|
||||
orchestrator_node_id: node_id,
|
||||
provider,
|
||||
..
|
||||
} = ctx;
|
||||
let mut pending_ready: Option<RuntimeReady> = None;
|
||||
let mut node_swim_started = false;
|
||||
|
|
@ -4271,6 +4291,7 @@ fn wait_for_weights_loaded(ctx: RuntimeReadyAckLoop<'_>, stage_index: u32) -> Re
|
|||
run_id,
|
||||
orchestrator_node_id: node_id,
|
||||
provider,
|
||||
..
|
||||
} = ctx;
|
||||
loop {
|
||||
pump(driver, stack, frame_tx);
|
||||
|
|
@ -4913,6 +4934,7 @@ fn serve_prompts(
|
|||
tokenizer_reply_to: ActorAddress,
|
||||
pipeline_plan: Option<&run_plan::RunPlan>,
|
||||
prompt_endpoint: EndpointAddr,
|
||||
swim_to_node: &BTreeMap<DistNodeId, u64>,
|
||||
) -> Result<(), String> {
|
||||
let RuntimeReadyAckLoop {
|
||||
driver,
|
||||
|
|
@ -4927,6 +4949,7 @@ fn serve_prompts(
|
|||
run_id,
|
||||
orchestrator_node_id: node_id,
|
||||
provider,
|
||||
orchestrator_actor,
|
||||
..
|
||||
} = ctx;
|
||||
let emit_prompt_evt =
|
||||
|
|
@ -4949,6 +4972,7 @@ fn serve_prompts(
|
|||
let mut active: Option<ActivePrompt> = None;
|
||||
loop {
|
||||
pump(driver, stack, frame_tx);
|
||||
orch_datastream.flush(dashboard, "orchestrator");
|
||||
if let Some(pipeline) = pipeline_runtime.as_mut() {
|
||||
pipeline.poll_driver(driver);
|
||||
pipeline.drain_tokenizer_events(
|
||||
|
|
@ -4971,6 +4995,21 @@ fn serve_prompts(
|
|||
)?;
|
||||
drain_frames(frame_rx, dashboard, orch_datastream);
|
||||
drain_orch_stdio_capture(orch_stdio_rx, orch_datastream, dashboard, run_id, node_id);
|
||||
let swim_transitions =
|
||||
emit_swim_transitions(orch_datastream, dashboard, run_id, node_id, stack);
|
||||
for transition in &swim_transitions {
|
||||
if transition.to == MemberState::Dead
|
||||
&& let Some(&lost_node_id) = swim_to_node.get(&transition.peer)
|
||||
{
|
||||
let _ = stack.runtime.send_to(
|
||||
orchestrator_actor,
|
||||
OrchestratorMsg::ObserveMembershipLost {
|
||||
run_id,
|
||||
node_id: lost_node_id,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
if stop_rx.try_recv().is_ok() {
|
||||
orch_datastream.emit_bootstrap(
|
||||
dashboard,
|
||||
|
|
@ -4980,6 +5019,10 @@ fn serve_prompts(
|
|||
"started",
|
||||
json!({"source":"stdin"}),
|
||||
);
|
||||
let _ = stack
|
||||
.runtime
|
||||
.send_to(orchestrator_actor, OrchestratorMsg::ObserveOperatorStop { run_id });
|
||||
pump(driver, stack, frame_tx);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
|
@ -5337,8 +5380,9 @@ fn emit_swim_transitions(
|
|||
run_id: u64,
|
||||
node_id: u64,
|
||||
stack: &DistributionRuntimeStack,
|
||||
) {
|
||||
for transition in stack.drain_swim_transitions() {
|
||||
) -> Vec<ObservedTransition> {
|
||||
let transitions = stack.drain_swim_transitions();
|
||||
for transition in &transitions {
|
||||
let peer = format!("{:?}", transition.peer);
|
||||
let from = transition.from.map(|state| format!("{:?}", state));
|
||||
let to = format!("{:?}", transition.to);
|
||||
|
|
@ -5366,8 +5410,9 @@ fn emit_swim_transitions(
|
|||
"member_state":member_state.clone(),
|
||||
}),
|
||||
);
|
||||
orch_datastream.emit_record(dashboard, &stack.membership_transition(&transition));
|
||||
orch_datastream.emit_record(dashboard, &stack.membership_transition(transition));
|
||||
}
|
||||
transitions
|
||||
}
|
||||
|
||||
fn emit_swim_probe_events(
|
||||
|
|
@ -5418,18 +5463,24 @@ fn local_tinygrad_worker_env(provider: &str) -> Option<(String, String)> {
|
|||
}
|
||||
|
||||
fn default_local_tinygrad_worker_path() -> Option<PathBuf> {
|
||||
let cwd_candidate = std::env::current_dir()
|
||||
.ok()
|
||||
.map(|cwd| cwd.join("apps").join("myelin-node").join("tinygrad_worker.py"));
|
||||
// The tinygrad worker script ships in the node-image build context at
|
||||
// `apps/myelin/node-image/tinygrad_worker.py` (see `chat/node_image.rs` and
|
||||
// the node-image Dockerfile). The process provider runs it directly via
|
||||
// `python3`, so resolve that path from the workspace cwd or this crate's
|
||||
// manifest dir. (Previously looked in `apps/myelin-node/`, a path left stale
|
||||
// by the `mvp-system` -> `myelin` app refactor and never present on disk.)
|
||||
let cwd_candidate = std::env::current_dir().ok().map(|cwd| {
|
||||
cwd.join("apps")
|
||||
.join("myelin")
|
||||
.join("node-image")
|
||||
.join("tinygrad_worker.py")
|
||||
});
|
||||
if let Some(candidate) = cwd_candidate.filter(|path| path.is_file()) {
|
||||
return Some(candidate.canonicalize().unwrap_or(candidate));
|
||||
}
|
||||
|
||||
let manifest_candidate = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("..")
|
||||
.join("..")
|
||||
.join("apps")
|
||||
.join("myelin-node")
|
||||
.join("node-image")
|
||||
.join("tinygrad_worker.py");
|
||||
manifest_candidate.is_file().then(|| {
|
||||
manifest_candidate
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ use std::time::{Duration, Instant};
|
|||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::config::RuntimeConfig;
|
||||
use swactor::runtime::{Ctx, Runtime};
|
||||
use swactor::stats::StatsHook;
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_transport::{CodecRegistry, CodecRemoteSink, NetworkMessage, TransportRouter};
|
||||
|
||||
|
|
@ -44,7 +45,6 @@ pub(crate) struct DistributionActorAddrs {
|
|||
}
|
||||
|
||||
pub(crate) struct DistributionRuntimeStack {
|
||||
pub node_id: NodeId,
|
||||
pub runtime: Arc<Runtime>,
|
||||
pub codec: Arc<CodecRegistry>,
|
||||
pub outbox: Outbox,
|
||||
|
|
@ -61,6 +61,7 @@ impl DistributionRuntimeStack {
|
|||
node_id: NodeId,
|
||||
config: DistributedNodeConfig,
|
||||
extend_codecs: impl FnOnce(&mut CodecRegistry),
|
||||
stats_hook: Option<Arc<dyn StatsHook>>,
|
||||
) -> Self {
|
||||
let mut runtime =
|
||||
Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()));
|
||||
|
|
@ -72,6 +73,9 @@ impl DistributionRuntimeStack {
|
|||
Arc::clone(&codec),
|
||||
Arc::clone(&transport_router),
|
||||
)));
|
||||
if let Some(hook) = stats_hook {
|
||||
runtime.set_stats_hook(hook);
|
||||
}
|
||||
let runtime = Arc::new(runtime);
|
||||
|
||||
let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
|
||||
|
|
@ -146,7 +150,6 @@ impl DistributionRuntimeStack {
|
|||
.expect("subscribe MembershipFanout");
|
||||
|
||||
Self {
|
||||
node_id,
|
||||
runtime,
|
||||
codec,
|
||||
outbox,
|
||||
|
|
|
|||
|
|
@ -72,12 +72,6 @@ impl StaticRelayProvider {
|
|||
parse_relay_url(raw).map(Self::new)
|
||||
}
|
||||
|
||||
pub(crate) fn from_env() -> Result<Option<Self>, String> {
|
||||
selected_relay_url_from_env()
|
||||
.map(|url| Self::from_url_str(&url).map(Some))
|
||||
.unwrap_or(Ok(None))
|
||||
}
|
||||
|
||||
pub(crate) fn url(&self) -> String {
|
||||
self.url.to_string()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,16 +16,6 @@ pub(crate) struct RunPlan {
|
|||
pub stages: Vec<StageRef>,
|
||||
}
|
||||
|
||||
impl RunPlan {
|
||||
pub(crate) fn test_linear(run_id: RunId, stages: Vec<StageRef>) -> Self {
|
||||
Self { run_id, stages }
|
||||
}
|
||||
|
||||
pub(crate) fn stage_nodes(&self) -> Vec<NodeId> {
|
||||
self.stages.iter().map(|stage| stage.node_id).collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RunConfig {
|
||||
pub run_id: RunId,
|
||||
|
|
@ -125,10 +115,6 @@ pub(crate) enum RunFaultReason {
|
|||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum LifecycleEvent {
|
||||
RunRejected {
|
||||
run_id: RunId,
|
||||
reason: RunFaultReason,
|
||||
},
|
||||
RunFaulted {
|
||||
run_id: RunId,
|
||||
reason: RunFaultReason,
|
||||
|
|
@ -175,8 +161,6 @@ pub(crate) enum RunCommand {
|
|||
},
|
||||
}
|
||||
|
||||
pub type OrchestratorHarness = OrchestratorRun;
|
||||
|
||||
pub(crate) struct OrchestratorRun {
|
||||
config: RunConfig,
|
||||
plan: Option<RunPlan>,
|
||||
|
|
|
|||
|
|
@ -165,7 +165,6 @@ pub(crate) enum EdgeKind {
|
|||
pub(crate) enum ObjectKind {
|
||||
Token,
|
||||
Activation,
|
||||
Weight,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
|
|||
|
|
@ -19,12 +19,6 @@ pub(crate) struct DeviceHandle {
|
|||
pub id: u64,
|
||||
}
|
||||
|
||||
impl DeviceHandle {
|
||||
pub(crate) fn new_current(id: u64) -> Self {
|
||||
Self { generation: 1, id }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct LayerRange {
|
||||
pub start: u32,
|
||||
|
|
@ -50,14 +44,6 @@ impl WeightSource {
|
|||
tokenizer,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn embedded_gguf(model_id: impl Into<String>, path: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
model_id,
|
||||
GgufSource::LocalPath(path.into()),
|
||||
TokenizerSource::EmbeddedGguf,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
@ -228,9 +214,6 @@ pub(crate) enum StageCommand {
|
|||
range: LayerRange,
|
||||
shard_plan: Option<StageShardPlan>,
|
||||
},
|
||||
RewireEdge {
|
||||
edge_id: EdgeId,
|
||||
},
|
||||
ExecuteStep(ExecuteStep),
|
||||
ReleaseInputHandle {
|
||||
object_id: ObjectId,
|
||||
|
|
@ -244,8 +227,6 @@ pub(crate) enum StageCommand {
|
|||
},
|
||||
}
|
||||
|
||||
pub type StageControllerHarness = StageController;
|
||||
|
||||
pub(crate) struct StageController {
|
||||
local_node_id: NodeId,
|
||||
provision: Option<ProvisionStage>,
|
||||
|
|
|
|||
50
apps/myelin/src/tests/harness.rs
Normal file
50
apps/myelin/src/tests/harness.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
//! Test-only constructors and harness type aliases for the `tests` tree.
|
||||
//!
|
||||
//! These helpers previously lived in the production modules
|
||||
//! (`orchestration/run_fsm`, `staging::control`) but serve only test code.
|
||||
//! Defining them here keeps them compiled under `#[cfg(test)]` only, so
|
||||
//! production builds stay free of the dead-code warnings they would otherwise
|
||||
//! trigger. Inherent methods resolve through their type, so existing call sites
|
||||
//! such as `fsm::RunPlan::test_linear(..)` and
|
||||
//! `stage::WeightSource::embedded_gguf(..)` work unchanged.
|
||||
|
||||
use crate::run_fsm::{NodeId as FsmNodeId, OrchestratorRun, RunId as FsmRunId, RunPlan, StageRef};
|
||||
use crate::run_plan::{GgufSource, TokenizerSource};
|
||||
use crate::staging::{DeviceHandle, StageController, WeightSource};
|
||||
|
||||
impl RunPlan {
|
||||
/// Build a linear pipeline plan (stage 0 -> stage 1 -> ... -> last stage).
|
||||
/// Test-only convenience over the public `RunPlan` fields.
|
||||
pub(crate) fn test_linear(run_id: FsmRunId, stages: Vec<StageRef>) -> Self {
|
||||
Self { run_id, stages }
|
||||
}
|
||||
|
||||
/// Node ids in declared stage order.
|
||||
pub(crate) fn stage_nodes(&self) -> Vec<FsmNodeId> {
|
||||
self.stages.iter().map(|stage| stage.node_id).collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl DeviceHandle {
|
||||
/// A handle on the worker's current (first) generation.
|
||||
pub(crate) fn new_current(id: u64) -> Self {
|
||||
Self { generation: 1, id }
|
||||
}
|
||||
}
|
||||
|
||||
impl WeightSource {
|
||||
/// Weights carried by an embedded GGUF file with an embedded tokenizer.
|
||||
pub(crate) fn embedded_gguf(model_id: impl Into<String>, path: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
model_id,
|
||||
GgufSource::LocalPath(path.into()),
|
||||
TokenizerSource::EmbeddedGguf,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Test alias for the orchestrator run core, retained for readable test prose.
|
||||
pub(crate) type OrchestratorHarness = OrchestratorRun;
|
||||
|
||||
/// Test alias for the stage controller core, retained for readable test prose.
|
||||
pub(crate) type StageControllerHarness = StageController;
|
||||
|
|
@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
|
|||
|
||||
use crate::run_fsm as fsm;
|
||||
use crate::run_plan as plan;
|
||||
use crate::tests::harness::OrchestratorHarness;
|
||||
use data_plane::edge_actor;
|
||||
use myelin::observability::lifecycle as obs;
|
||||
use myelin::orchestration::engine_builder as engine;
|
||||
|
|
@ -34,7 +35,7 @@ pub struct LocalMockCluster {
|
|||
engine_events: Vec<engine::EngineEvent>,
|
||||
plan: plan::RunPlan,
|
||||
nodes: BTreeMap<u32, MockNode>,
|
||||
orchestrator: Option<fsm::OrchestratorHarness>,
|
||||
orchestrator: Option<OrchestratorHarness>,
|
||||
orchestrator_command_cursor: usize,
|
||||
orchestrator_event_cursor: usize,
|
||||
trace: Vec<obs::Event>,
|
||||
|
|
@ -303,7 +304,7 @@ impl LocalMockCluster {
|
|||
self.orchestrator_command_cursor = 0;
|
||||
self.orchestrator_event_cursor = 0;
|
||||
self.scenario = scenario;
|
||||
self.orchestrator = Some(fsm::OrchestratorHarness::new(fsm::RunConfig {
|
||||
self.orchestrator = Some(OrchestratorHarness::new(fsm::RunConfig {
|
||||
run_id: fsm::RunId(self.run_id.0),
|
||||
max_tokens: u64::from(self.max_tokens),
|
||||
prompt: tokenize(prompt),
|
||||
|
|
@ -742,7 +743,6 @@ impl LocalMockCluster {
|
|||
self.push_run(obs::EventKind::RunCompleted, obs::Component::Orchestrator);
|
||||
}
|
||||
fsm::LifecycleEvent::RunFaulted { .. }
|
||||
| fsm::LifecycleEvent::RunRejected { .. }
|
||||
| fsm::LifecycleEvent::RunOperatorStopped { .. } => {
|
||||
self.push_run(obs::EventKind::RunFaulted, obs::Component::Orchestrator);
|
||||
}
|
||||
|
|
@ -852,7 +852,7 @@ impl LocalMockCluster {
|
|||
.count()
|
||||
}
|
||||
|
||||
fn orchestrator_mut(&mut self) -> &mut fsm::OrchestratorHarness {
|
||||
fn orchestrator_mut(&mut self) -> &mut OrchestratorHarness {
|
||||
self.orchestrator
|
||||
.as_mut()
|
||||
.expect("run_prompt must initialize orchestrator")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::run_plan as plan;
|
||||
use data_plane::edge_actor;
|
||||
use myelin::staging as stage;
|
||||
use crate::tests::harness::StageControllerHarness;
|
||||
|
||||
use super::mock_transport::MockObject;
|
||||
use super::mock_worker::MockWorker;
|
||||
|
|
@ -16,7 +17,7 @@ pub struct MockNode {
|
|||
inbound_edge: Option<plan::EdgeId>,
|
||||
outbound_edge: Option<plan::EdgeId>,
|
||||
outbound_object_allocator: Option<edge_actor::ObjectIdAllocator>,
|
||||
controller: stage::StageControllerHarness,
|
||||
controller: StageControllerHarness,
|
||||
worker: MockWorker,
|
||||
event_cursor: usize,
|
||||
}
|
||||
|
|
@ -34,7 +35,7 @@ impl MockNode {
|
|||
inbound_edge: None,
|
||||
outbound_edge: None,
|
||||
outbound_object_allocator: None,
|
||||
controller: stage::StageControllerHarness::new(stage::NodeId(node_id.0)),
|
||||
controller: StageControllerHarness::new(stage::NodeId(node_id.0)),
|
||||
worker: MockWorker::new(stage_index, eos_after_sequence),
|
||||
event_cursor: 0,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
mod harness;
|
||||
mod local_e2e_guarantees;
|
||||
mod local_mock;
|
||||
mod node_guarantees;
|
||||
|
|
|
|||
|
|
@ -502,6 +502,7 @@ mod run_fsm {
|
|||
//! `specs/BEHAVIOR_GUARANTEES.md`.
|
||||
|
||||
use crate::run_fsm as fsm;
|
||||
use crate::tests::harness::OrchestratorHarness;
|
||||
|
||||
// A three-stage plan proves multi-stage provisioning and readiness without
|
||||
// making tests depend on any placement heuristic. The plan is already valid;
|
||||
|
|
@ -529,8 +530,8 @@ mod run_fsm {
|
|||
// The harness is the black-box public boundary for the run FSM. It accepts
|
||||
// observable events and records emitted commands/events; tests never inspect an
|
||||
// internal FSM enum or private readiness counter.
|
||||
fn new_run() -> fsm::OrchestratorHarness {
|
||||
fsm::OrchestratorHarness::new(fsm::RunConfig {
|
||||
fn new_run() -> OrchestratorHarness {
|
||||
OrchestratorHarness::new(fsm::RunConfig {
|
||||
run_id: fsm::RunId(7),
|
||||
max_tokens: 4,
|
||||
prompt: vec![101, 102, 103],
|
||||
|
|
@ -680,7 +681,6 @@ mod run_fsm {
|
|||
});
|
||||
assert!(invalid.events().iter().any(|event| {
|
||||
matches!(event, fsm::LifecycleEvent::RunFaulted { .. })
|
||||
|| matches!(event, fsm::LifecycleEvent::RunRejected { .. })
|
||||
}));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ mod stage_controller {
|
|||
//! `specs/BEHAVIOR_GUARANTEES.md`.
|
||||
|
||||
use myelin::staging as stage;
|
||||
use crate::tests::harness::StageControllerHarness;
|
||||
|
||||
// This provision fixture represents a single middle stage. It has one inbound
|
||||
// and one outbound edge so tests can prove the controller uses assigned edges
|
||||
|
|
@ -37,8 +38,8 @@ mod stage_controller {
|
|||
// The harness exposes only public messages. Tests intentionally do not inspect
|
||||
// private controller states such as "Preparing" or "Executing"; they infer
|
||||
// controller behavior from emitted commands and lifecycle events.
|
||||
fn new_controller() -> stage::StageControllerHarness {
|
||||
stage::StageControllerHarness::new(stage::NodeId(11))
|
||||
fn new_controller() -> StageControllerHarness {
|
||||
StageControllerHarness::new(stage::NodeId(11))
|
||||
}
|
||||
|
||||
// Preparation readiness has four independent prerequisites. Listing them as
|
||||
|
|
@ -60,7 +61,7 @@ mod stage_controller {
|
|||
// This helper provisions and readies a stage through public events. Tests that
|
||||
// focus on execution use it to avoid duplicating setup while still going through
|
||||
// the same observable path as production.
|
||||
fn ready_stage() -> stage::StageControllerHarness {
|
||||
fn ready_stage() -> StageControllerHarness {
|
||||
let mut harness = new_controller();
|
||||
harness.observe(stage::StageEvent::ProvisionStage {
|
||||
from: stage::NodeId(99),
|
||||
|
|
@ -103,15 +104,6 @@ mod stage_controller {
|
|||
)
|
||||
}));
|
||||
|
||||
// The controller must not emit any command that replaces the provisioned
|
||||
// edge ids with a locally chosen edge.
|
||||
assert!(
|
||||
!harness
|
||||
.commands()
|
||||
.iter()
|
||||
.any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) })
|
||||
);
|
||||
|
||||
// An unauthorized provision attempt must fault before setup can begin.
|
||||
let mut unauthorized = new_controller();
|
||||
unauthorized.observe(stage::StageEvent::ProvisionStage {
|
||||
|
|
|
|||
|
|
@ -8,4 +8,4 @@ Keep this crate read-only with respect to observed programs.
|
|||
- It must not send control signals to observed runtimes.
|
||||
- It must not require changes outside `crates/dashboard` for dashboard-only work.
|
||||
|
||||
Main built-in view: `/view/swactor/workers`, backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present.
|
||||
Main built-in views: `/view/swactor/workers` (worker-centric) and `/view/swactor/actor-overview` + `/view/swactor/actor-dossier` (actor-centric), all backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present. Actor views are pure frame consumers and tolerant of publisher shape.
|
||||
|
|
|
|||
|
|
@ -10,5 +10,5 @@ swactor = { path = "../..", features = ["serde"] }
|
|||
parking_lot = "0.12"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tokio = { version = "1", features = ["net", "rt-multi-thread", "sync"] }
|
||||
tokio.workspace = true
|
||||
tokio-stream = "0.1"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Read-only HTML/SSE dashboard over incoming datastream frames.
|
||||
|
||||
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor worker page is hosted here because worker/actor/message processing is universal to swactor programs.
|
||||
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor views are hosted here because worker/actor/message processing is universal to swactor programs: the worker page, and the actor overview (fused roster) plus per-actor dossier.
|
||||
|
||||
## Routes
|
||||
|
||||
|
|
@ -16,5 +16,9 @@ The crate owns the Axum server, bounded raw frame window, and view registry. Com
|
|||
- `GET /api/view/fleet` — fleet and machine telemetry JSON snapshot
|
||||
- `GET /view/swactor/workers` — built-in worker page
|
||||
- `GET /api/view/swactor/workers` — worker page JSON snapshot
|
||||
- `GET /view/swactor/actor-overview` — built-in actor overview + roster page
|
||||
- `GET /api/view/swactor/actor-overview` — actor overview JSON snapshot
|
||||
- `GET /view/swactor/actor-dossier` — built-in per-actor dossier page
|
||||
- `GET /api/view/swactor/actor-dossier` — actor dossier JSON snapshot
|
||||
|
||||
All state is derived from observed frames. The dashboard sends no control signals back to producers.
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ impl DashboardHandle {
|
|||
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
|
||||
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
|
||||
views.register(swactor::worker_view());
|
||||
views.register(swactor::actor_overview_view());
|
||||
views.register(swactor::actor_dossier_view());
|
||||
let store = Arc::new(DashboardStore::new(
|
||||
config.raw_frame_history,
|
||||
Arc::clone(&views),
|
||||
|
|
|
|||
185
crates/dashboard/src/swactor/actor_dossier.html
Normal file
185
crates/dashboard/src/swactor/actor_dossier.html
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Actor dossier — swactor</title>
|
||||
<style>
|
||||
:root{
|
||||
--bg:#0f172a; --panel:#1e293b; --panel2:#172033; --border:#334155;
|
||||
--text:#e2e8f0; --muted:#94a3b8; --accent:#60a5fa;
|
||||
--running:#34d399; --poisoned:#f87171;
|
||||
font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;
|
||||
}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;padding:0;background:var(--bg);color:var(--text);}
|
||||
.mono,.table,th,td,.value,.num{font-variant-numeric:tabular-nums;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;}
|
||||
header.bar{display:flex;align-items:center;gap:16px;padding:14px 20px;border-bottom:1px solid var(--border);background:var(--panel);position:sticky;top:0;z-index:5;}
|
||||
header.bar h1{font-size:18px;margin:0;font-weight:600;letter-spacing:.01em;}
|
||||
nav.tabs{display:flex;gap:4px;margin-left:auto;}
|
||||
nav.tabs a{padding:6px 12px;border-radius:8px;color:var(--muted);text-decoration:none;font-size:13px;border:1px solid transparent;}
|
||||
nav.tabs a:hover{color:var(--text);background:var(--panel2);}
|
||||
nav.tabs a.cur{color:var(--accent);background:var(--panel2);border-color:var(--border);}
|
||||
select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
|
||||
.wrap{padding:20px;max-width:1100px;margin:0 auto;}
|
||||
.picker{display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin-bottom:18px;}
|
||||
.picker label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em;}
|
||||
#pick{min-width:300px;max-width:560px;flex:1;}
|
||||
.back-link{font-size:13px;}
|
||||
.badge{display:inline-block;padding:2px 8px;border-radius:99px;font-size:11px;text-transform:uppercase;letter-spacing:.06em;font-weight:600;}
|
||||
.empty{padding:28px;border:1px dashed #475569;border-radius:12px;color:var(--muted);text-align:center;}
|
||||
.toast{position:fixed;left:50%;bottom:20px;transform:translateX(-50%);background:var(--panel2);border:1px solid var(--border);color:var(--text);padding:10px 16px;border-radius:10px;font-size:13px;max-width:min(90vw,560px);text-align:center;z-index:50;box-shadow:0 8px 24px rgba(0,0,0,.45);}
|
||||
.toast.err{border-color:var(--poisoned);color:var(--poisoned);}
|
||||
.kvs{display:grid;grid-template-columns:max-content 1fr;gap:6px 14px;}
|
||||
.kvs .k{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.05em;}
|
||||
.kvs .v{font-size:13px;word-break:break-word;}
|
||||
.section{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:16px 18px;margin-bottom:16px;}
|
||||
h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:0 0 12px;}
|
||||
.mb-dyn{display:flex;gap:24px;align-items:center;flex-wrap:wrap;}
|
||||
.mb-spark{flex:1 1 320px;min-width:0;}
|
||||
.mb-spark .spark{display:block;width:100%;height:120px;}
|
||||
.mb-vals{display:flex;flex-direction:column;gap:16px;min-width:150px;}
|
||||
.mb-val .label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em;}
|
||||
.mb-val .value{font-size:28px;margin-top:4px;font-weight:600;line-height:1;}
|
||||
.bar-row{display:flex;align-items:center;gap:8px;margin:5px 0;}
|
||||
.bar-row .name{width:48%;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}
|
||||
.bar-track{flex:1;height:10px;background:var(--panel2);border-radius:99px;overflow:hidden;}
|
||||
.bar-fill{height:100%;background:var(--accent);border-radius:99px;}
|
||||
.bar-row .num{width:90px;text-align:right;color:var(--muted);}
|
||||
.bar-row.last .bar-fill{background:var(--running);}
|
||||
.bar-row.last{font-weight:600;}
|
||||
a{color:var(--accent);text-decoration:none;}
|
||||
.stale{opacity:.55;}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header class="bar">
|
||||
<h1>Actor dossier</h1>
|
||||
<select id="runtime"></select>
|
||||
<nav class="tabs">
|
||||
<a href="/view/swactor/actor-overview">Overview</a>
|
||||
<a href="/view/swactor/actor-dossier" class="cur">Dossier</a>
|
||||
<a href="/view/swactor/workers">Workers</a>
|
||||
</nav>
|
||||
</header>
|
||||
<div class="wrap">
|
||||
<div class="picker">
|
||||
<label for="pick">Actor</label>
|
||||
<select id="pick"></select>
|
||||
<a class="back-link" href="/view/swactor/actor-overview">← back to overview</a>
|
||||
</div>
|
||||
<div id="empty" class="toast" hidden>No runtime actor frames yet.</div>
|
||||
<div id="content"></div>
|
||||
</div>
|
||||
<script>
|
||||
const POLL_MS=750;
|
||||
const STATE_COLOR={running:'#34d399',poisoned:'#f87171'};
|
||||
const elRuntime=document.getElementById('runtime');
|
||||
const elPick=document.getElementById('pick');
|
||||
const elEmpty=document.getElementById('empty');
|
||||
const elContent=document.getElementById('content');
|
||||
let runtimes=[],runtimeKey='',selectedAddr='';
|
||||
|
||||
function fmt(n,d=0){if(n==null||n===undefined)return'\u2014';if(typeof n!=='number')return String(n);return n.toLocaleString(undefined,{maximumFractionDigits:d});}
|
||||
function esc(s){return String(s==null?'':s).replace(/[&<>"]/g,function(c){return{'&':'&','<':'<','>':'>','"':'"'}[c];});}
|
||||
function shortAddr(a){a=String(a||'');return a.length<=12?a:a.slice(0,6)+'\u2026'+a.slice(-4);}
|
||||
function urlActor(){try{return new URLSearchParams(location.search).get('actor');}catch(e){return null;}}
|
||||
function stateBadge(a){const s=a.poisoned?'poisoned':'running';const c=STATE_COLOR[s];return '<span class="badge" style="color:'+c+';background:'+c+'22">'+s+'</span>';}
|
||||
function growthColor(g){if(g>1.5)return '#f87171';if(g>0.3)return '#fbbf24';return '#34d399';}
|
||||
function areaChart(data,w,h,color){
|
||||
if(!data||data.length<2)return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="100%" height="'+h+'" preserveAspectRatio="none"></svg>';
|
||||
const max=Math.max(1,...data);const n=data.length;
|
||||
let line='',area='0,'+h+' ';
|
||||
for(let i=0;i<n;i++){const x=(i/(n-1)*w).toFixed(1);const y=(h-(data[i]/max)*(h-2)-1).toFixed(1);line+=x+','+y+' ';area+=x+','+y+' ';}
|
||||
area+=' '+w+','+h;
|
||||
return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="100%" height="'+h+'" preserveAspectRatio="none">'+
|
||||
'<polygon points="'+area.trim()+'" fill="'+(color||'#60a5fa')+'22"/>'+
|
||||
'<polyline points="'+line.trim()+'" fill="none" stroke="'+(color||'#60a5fa')+'" stroke-width="1.5"/></svg>';
|
||||
}
|
||||
function kv(k,vhtml){return '<div class="k">'+esc(k)+'</div><div class="v">'+vhtml+'</div>';}
|
||||
function typeShort(t){t=String(t||'');const i=t.lastIndexOf('::');return i>=0?t.slice(i+2):t;}
|
||||
function pickRuntime(list){
|
||||
if(!runtimeKey||!list.some(r=>r.stream.key===runtimeKey)){
|
||||
const live=list.find(r=>r.live)||list[0];runtimeKey=live?live.stream.key:'';
|
||||
}
|
||||
return list.find(r=>r.stream.key===runtimeKey)||null;
|
||||
}
|
||||
function currentActors(rt){return rt?(rt.actors||[]):[];}
|
||||
function findActor(rt){
|
||||
const actors=currentActors(rt);
|
||||
if(selectedAddr){const hit=actors.find(a=>a.address===selectedAddr);if(hit)return hit;}
|
||||
return actors[0]||null;
|
||||
}
|
||||
function renderPicker(rt){
|
||||
const actors=currentActors(rt);
|
||||
elPick.innerHTML=actors.map(a=>
|
||||
'<option value="'+esc(a.address)+'">'+esc(shortAddr(a.address))+(a.name?' \u00b7 '+esc(a.name):'')+'</option>'
|
||||
).join('');
|
||||
const cur=findActor(rt);
|
||||
if(cur){selectedAddr=cur.address;elPick.value=selectedAddr;}
|
||||
}
|
||||
function renderRuntimeSelector(){
|
||||
elRuntime.innerHTML=runtimes.map(r=>
|
||||
'<option value="'+esc(r.stream.key)+'">'+esc(r.stream.node)+'#'+r.stream.life+(r.live?'':' (stale)')+'</option>'
|
||||
).join('');
|
||||
if(runtimeKey)elRuntime.value=runtimeKey;
|
||||
}
|
||||
function render(a,rt){
|
||||
if(!a){elContent.innerHTML='<div class="empty">Select an actor. None are present on the current frame.</div>';return;}
|
||||
const lastType=a.last_msg_type||'\u2014';
|
||||
const diet=(a.message_type_counts||[]).slice();
|
||||
diet.sort((x,y)=>y.count-x.count);
|
||||
const dmax=Math.max(1,...diet.map(d=>d.count));
|
||||
const lastName=a.last_msg_type;
|
||||
const dietHtml=diet.length?diet.map(d=>{
|
||||
const cls=d.message_type===lastName?'bar-row last':'bar-row';
|
||||
return '<div class="'+cls+'"><span class="name">'+esc(d.message_type)+'</span>'+
|
||||
'<div class="bar-track"><div class="bar-fill" style="width:'+(d.count/dmax*100).toFixed(1)+'%"></div></div>'+
|
||||
'<span class="num">'+fmt(d.count)+'</span></div>';
|
||||
}).join(''):'<div class="empty">no message-type counts on frame</div>';
|
||||
const hist=(a.history||[]).map(h=>h.mailbox_depth);
|
||||
const g=a.mailbox_growth||0;
|
||||
elContent.innerHTML=
|
||||
'<div class="section"><h2>Identity & lifecycle</h2><div class="kvs">'+
|
||||
kv('address','<span class="mono">'+esc(a.address)+'</span>')+
|
||||
(a.name?kv('name',esc(a.name)):'')+
|
||||
(a.actor_type?kv('actor type','<span class="mono" title="'+esc(a.actor_type)+'">'+esc(typeShort(a.actor_type))+'</span>'):'')+
|
||||
(a.message_type?kv('message type','<span class="mono" title="'+esc(a.message_type)+'">'+esc(typeShort(a.message_type))+'</span>'):'')+
|
||||
kv('state',stateBadge(a))+
|
||||
kv('worker',a.worker_id==null?'\u2014':esc(a.worker_id))+
|
||||
kv('last message',esc(lastType))+
|
||||
kv('last seen',a.last_seen_ms_ago!=null?(a.last_seen_ms_ago/1000).toFixed(1)+'s ago':'\u2014')+
|
||||
'</div></div>'+
|
||||
'<div class="section"><h2>Mailbox dynamics</h2><div class="mb-dyn">'+
|
||||
'<div class="mb-spark">'+areaChart(hist,720,120,growthColor(g))+'</div>'+
|
||||
'<div class="mb-vals">'+
|
||||
'<div class="mb-val"><div class="label">depth</div><div class="value" style="color:'+growthColor(g)+'">'+fmt(a.mailbox_depth)+'</div></div>'+
|
||||
'<div class="mb-val"><div class="label">growth</div><div class="value">'+fmt(g,1)+'/s</div></div>'+
|
||||
'<div class="mb-val"><div class="label">msg/s</div><div class="value">'+fmt(a.msg_per_sec,1)+'</div></div>'+
|
||||
'</div>'+
|
||||
'</div></div>'+
|
||||
'<div class="section"><h2>Message diet</h2>'+dietHtml+'</div>';
|
||||
}
|
||||
async function refresh(){
|
||||
try{
|
||||
const res=await fetch('/api/view/swactor/actor-dossier',{cache:'no-store'});
|
||||
if(!res.ok)throw new Error('HTTP '+res.status);
|
||||
const data=await res.json();
|
||||
runtimes=data.runtimes||[];
|
||||
renderRuntimeSelector();
|
||||
const rt=pickRuntime(runtimes);
|
||||
if(rt&¤tActors(rt).length>0){elEmpty.hidden=true;}else{elEmpty.className='toast';elEmpty.textContent='No runtime actor frames yet.';elEmpty.hidden=false;}
|
||||
renderPicker(rt);
|
||||
render(findActor(rt),rt);
|
||||
}catch(e){
|
||||
elEmpty.className='toast err';elEmpty.hidden=false;elEmpty.textContent='Failed to load actor frames: '+e.message;elContent.innerHTML='';
|
||||
}
|
||||
}
|
||||
elRuntime.addEventListener('change',e=>{runtimeKey=e.target.value;const rt=pickRuntime(runtimes);renderPicker(rt);render(findActor(rt),rt);});
|
||||
elPick.addEventListener('change',e=>{selectedAddr=e.target.value;const rt=pickRuntime(runtimes);render(findActor(rt),rt);});
|
||||
const q=urlActor();if(q)selectedAddr=q;
|
||||
refresh();
|
||||
setInterval(refresh,POLL_MS);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
312
crates/dashboard/src/swactor/actor_overview.html
Normal file
312
crates/dashboard/src/swactor/actor_overview.html
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Actor overview — swactor</title>
|
||||
<style>
|
||||
:root{
|
||||
--bg:#0f172a; --panel:#1e293b; --panel2:#172033; --border:#334155;
|
||||
--text:#e2e8f0; --muted:#94a3b8; --accent:#60a5fa;
|
||||
--running:#34d399; --poisoned:#f87171;
|
||||
font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;
|
||||
}
|
||||
*{box-sizing:border-box}
|
||||
body{margin:0;padding:0;background:var(--bg);color:var(--text);}
|
||||
.mono,.table,th,td,.value,.num{font-variant-numeric:tabular-nums;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;}
|
||||
header.bar{display:flex;align-items:center;gap:16px;padding:14px 20px;border-bottom:1px solid var(--border);background:var(--panel);position:sticky;top:0;z-index:5;}
|
||||
header.bar h1{font-size:18px;margin:0;font-weight:600;letter-spacing:.01em;}
|
||||
nav.tabs{display:flex;gap:4px;margin-left:auto;}
|
||||
nav.tabs a{padding:6px 12px;border-radius:8px;color:var(--muted);text-decoration:none;font-size:13px;border:1px solid transparent;}
|
||||
nav.tabs a:hover{color:var(--text);background:var(--panel2);}
|
||||
nav.tabs a.cur{color:var(--accent);background:var(--panel2);border-color:var(--border);}
|
||||
select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
|
||||
.wrap{padding:20px;max-width:1400px;margin:0 auto;}
|
||||
.grid{display:grid;gap:12px;}
|
||||
.cards{grid-template-columns:repeat(auto-fit,minmax(150px,1fr));margin-bottom:16px;}
|
||||
.card{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:14px;}
|
||||
.card .label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em;}
|
||||
.card .value{font-size:26px;margin-top:6px;font-weight:600;}
|
||||
.card .value.bad{color:var(--poisoned);}
|
||||
.badge{display:inline-block;padding:2px 8px;border-radius:99px;font-size:11px;text-transform:uppercase;letter-spacing:.06em;font-weight:600;}
|
||||
.table{width:100%;border-collapse:collapse;background:var(--panel);border:1px solid var(--border);border-radius:12px;overflow:hidden;}
|
||||
.table th,.table td{padding:8px 10px;border-bottom:1px solid #1f2937;text-align:left;white-space:nowrap;}
|
||||
.table th{color:#93c5fd;background:var(--panel2);cursor:pointer;user-select:none;font-size:12px;text-transform:uppercase;letter-spacing:.05em;}
|
||||
.table th:hover{color:#fff;}
|
||||
.table tr{cursor:pointer;}
|
||||
.table tbody tr:hover{background:#1d4ed822;}
|
||||
.table td.muted,.muted{color:var(--muted);}
|
||||
.spark{display:block;}
|
||||
.bar-row{display:flex;align-items:center;gap:8px;margin:5px 0;}
|
||||
.bar-row .name{width:38%;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}
|
||||
.bar-track{flex:1;height:10px;background:var(--panel2);border-radius:99px;overflow:hidden;}
|
||||
.bar-fill{height:100%;background:var(--accent);border-radius:99px;}
|
||||
.bar-row .num{width:90px;text-align:right;color:var(--muted);}
|
||||
input{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
|
||||
.filters{display:flex;gap:10px;flex-wrap:wrap;align-items:center;margin-bottom:14px;}
|
||||
.filters label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em;}
|
||||
h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:22px 0 10px;}
|
||||
.empty{padding:28px;border:1px dashed #475569;border-radius:12px;color:var(--muted);text-align:center;}
|
||||
.toast{position:fixed;left:50%;bottom:20px;transform:translateX(-50%);background:var(--panel2);border:1px solid var(--border);color:var(--text);padding:10px 16px;border-radius:10px;font-size:13px;max-width:min(90vw,560px);text-align:center;z-index:50;box-shadow:0 8px 24px rgba(0,0,0,.45);}
|
||||
.toast.err{border-color:var(--poisoned);color:var(--poisoned);}
|
||||
.stacked{display:flex;height:18px;border-radius:6px;overflow:hidden;border:1px solid var(--border);}
|
||||
.legend{display:flex;gap:14px;flex-wrap:wrap;margin-top:8px;font-size:12px;color:var(--muted);}
|
||||
.legend i{display:inline-block;width:12px;height:12px;margin-right:5px;border-radius:3px;vertical-align:middle;}
|
||||
a{color:var(--accent);text-decoration:none;}
|
||||
.roster-box{max-height:520px;overflow:auto;border:1px solid var(--border);border-radius:12px;}
|
||||
.roster-box .table{border-radius:0;border:none;}
|
||||
.stale{opacity:.55;}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header class="bar">
|
||||
<h1>Actor overview</h1>
|
||||
<select id="runtime"></select>
|
||||
<nav class="tabs">
|
||||
<a href="/view/swactor/actor-overview" class="cur">Overview</a>
|
||||
<a href="/view/swactor/actor-dossier">Dossier</a>
|
||||
<a href="/view/swactor/workers">Workers</a>
|
||||
</nav>
|
||||
</header>
|
||||
<div class="wrap">
|
||||
<div id="empty" class="toast" hidden>No runtime actor frames yet. A live swactor runtime publishing <code>runtime.actors</code> will populate this page.</div>
|
||||
<div id="cards" class="grid cards"></div>
|
||||
|
||||
<h2>State distribution</h2>
|
||||
<div class="stacked" id="dist"></div>
|
||||
<div class="legend" id="distLegend"></div>
|
||||
|
||||
<h2>Throughput</h2>
|
||||
<div id="throughput"></div>
|
||||
|
||||
<h2>Worker balance</h2>
|
||||
<div id="workers"></div>
|
||||
|
||||
<h2>Hot actors — mailbox growth</h2>
|
||||
<div id="hot"></div>
|
||||
|
||||
<h2>Alerts</h2>
|
||||
<div id="alerts"></div>
|
||||
|
||||
<h2>Roster</h2>
|
||||
<div class="filters">
|
||||
<label for="f-search">search</label>
|
||||
<input id="f-search" type="search" placeholder="name · addr · worker" autocomplete="off" spellcheck="false">
|
||||
<label for="f-state">state</label>
|
||||
<select id="f-state">
|
||||
<option value="all">all states</option>
|
||||
<option value="running">running</option>
|
||||
<option value="poisoned">poisoned</option>
|
||||
</select>
|
||||
<label for="f-worker">worker</label>
|
||||
<select id="f-worker"></select>
|
||||
</div>
|
||||
<div class="roster-box">
|
||||
<table class="table">
|
||||
<thead><tr id="head"></tr></thead>
|
||||
<tbody id="rows"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const POLL_MS=750;
|
||||
const elRuntime=document.getElementById('runtime');
|
||||
const elEmpty=document.getElementById('empty');
|
||||
const elCards=document.getElementById('cards');
|
||||
const elDist=document.getElementById('dist');
|
||||
const elDistLegend=document.getElementById('distLegend');
|
||||
const elThroughput=document.getElementById('throughput');
|
||||
const elWorkers=document.getElementById('workers');
|
||||
const elHot=document.getElementById('hot');
|
||||
const elAlerts=document.getElementById('alerts');
|
||||
const elHead=document.getElementById('head');
|
||||
const elRows=document.getElementById('rows');
|
||||
const elSearch=document.getElementById('f-search');
|
||||
const elState=document.getElementById('f-state');
|
||||
const elWorker=document.getElementById('f-worker');
|
||||
|
||||
const STATE_COLOR={running:'#34d399',poisoned:'#f87171'};
|
||||
const COLS=[['actor','Actor'],['type','Type'],['state','State'],['mailbox','Mailbox'],['rate','Msg/s'],
|
||||
['processed','Processed'],['last','Last msg'],['worker','Worker']];
|
||||
const DEFDIR={actor:'asc',type:'asc',state:'asc',mailbox:'desc',rate:'desc',processed:'desc',last:'asc',worker:'asc'};
|
||||
let sortKey='mailbox',sortDir='desc';
|
||||
let fSearch='',fState='all',fWorker='all';
|
||||
let runtimes=[],runtimeKey='';
|
||||
|
||||
function fmt(n,d=0){if(n==null||n===undefined)return'\u2014';if(typeof n!=='number')return String(n);return n.toLocaleString(undefined,{maximumFractionDigits:d});}
|
||||
function ago(sec){sec=Math.max(0,Math.round(sec));if(sec<60)return sec+'s';const m=Math.floor(sec/60);if(m<60)return m+'m';return Math.floor(m/60)+'h';}
|
||||
function esc(s){return String(s==null?'':s).replace(/[&<>"]/g,function(c){return{'&':'&','<':'<','>':'>','"':'"'}[c];});}
|
||||
function shortAddr(a){a=String(a||'');return a.length<=12?a:a.slice(0,6)+'\u2026'+a.slice(-4);}
|
||||
function typeShort(t){t=String(t||'');const i=t.lastIndexOf('::');return i>=0?t.slice(i+2):t;}
|
||||
function stateBadge(a){const s=a.poisoned?'poisoned':'running';const c=STATE_COLOR[s];return '<span class="badge" style="color:'+c+';background:'+c+'22">'+s+'</span>';}
|
||||
function growthColor(g){if(g>1.5)return '#f87171';if(g>0.3)return '#fbbf24';return '#34d399';}
|
||||
function sparkline(data,w,h,color){
|
||||
if(!data||data.length<2)return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="'+w+'" height="'+h+'"></svg>';
|
||||
const max=Math.max(1,...data);const n=data.length;
|
||||
let pts='';for(let i=0;i<n;i++)pts+=(i/(n-1)*w).toFixed(1)+','+(h-(data[i]/max)*(h-2)-1).toFixed(1)+' ';
|
||||
return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="'+w+'" height="'+h+'" preserveAspectRatio="none"><polyline points="'+pts.trim()+'" fill="none" stroke="'+(color||'#60a5fa')+'" stroke-width="1.5"/></svg>';
|
||||
}
|
||||
function pickRuntime(list){
|
||||
if(!runtimeKey||!list.some(r=>r.stream.key===runtimeKey)){
|
||||
const live=list.find(r=>r.live)||list[0];
|
||||
runtimeKey=live?live.stream.key:'';
|
||||
}
|
||||
return list.find(r=>r.stream.key===runtimeKey)||null;
|
||||
}
|
||||
function renderRuntimeSelector(){
|
||||
elRuntime.innerHTML=runtimes.map(r=>
|
||||
'<option value="'+esc(r.stream.key)+'">'+esc(r.stream.node)+'#'+r.stream.life+(r.live?'':' (stale)')+'</option>'
|
||||
).join('');
|
||||
if(runtimeKey)elRuntime.value=runtimeKey;
|
||||
}
|
||||
function renderCards(rt){
|
||||
const s=rt.summary||{};
|
||||
const uptime=s.uptime_ms!=null?ago(s.uptime_ms/1000):'\u2014';
|
||||
elCards.innerHTML=
|
||||
'<div class="card"><div class="label">Actors</div><div class="value">'+fmt(s.actors)+'</div></div>'+
|
||||
'<div class="card"><div class="label">msg/s</div><div class="value">'+fmt(s.msg_per_sec,1)+'</div></div>'+
|
||||
'<div class="card"><div class="label">Mailbox</div><div class="value">'+fmt(s.mailbox_depth)+'</div></div>'+
|
||||
'<div class="card"><div class="label">Poisoned</div><div class="value'+(s.poisoned>0?' bad':'')+'">'+fmt(s.poisoned)+'</div></div>'+
|
||||
'<div class="card"><div class="label">Uptime</div><div class="value">'+uptime+'</div></div>';
|
||||
}
|
||||
function renderDist(rt){
|
||||
const actors=rt.actors||[];const n=actors.length||1;
|
||||
let running=0,poisoned=0;
|
||||
actors.forEach(a=>{if(a.poisoned)poisoned++;else running++;});
|
||||
let segs='';
|
||||
if(running)segs+='<div style="width:'+(running/n*100).toFixed(2)+'%;background:'+STATE_COLOR.running+';"></div>';
|
||||
if(poisoned)segs+='<div style="width:'+(poisoned/n*100).toFixed(2)+'%;background:'+STATE_COLOR.poisoned+';"></div>';
|
||||
elDist.innerHTML=segs||'<div style="width:100%;background:var(--panel2);"></div>';
|
||||
elDistLegend.innerHTML='<span><i style="background:'+STATE_COLOR.running+'"></i>running: '+running+'</span>'+
|
||||
'<span><i style="background:'+STATE_COLOR.poisoned+'"></i>poisoned: '+poisoned+'</span>';
|
||||
}
|
||||
function renderThroughput(rt){
|
||||
const h=(rt.history||[]).map(x=>x.msg_per_sec);
|
||||
const mb=(rt.history||[]).map(x=>x.mailbox_depth);
|
||||
elThroughput.innerHTML=
|
||||
sparkline(h,760,90,'#60a5fa')+'<div style="height:6px"></div>'+
|
||||
sparkline(mb,760,90,'#fbbf24')+
|
||||
'<div class="legend"><span><i style="background:#60a5fa"></i>msg/s</span><span><i style="background:#fbbf24"></i>mailbox</span></div>';
|
||||
}
|
||||
function renderWorkers(rt){
|
||||
const counts={};
|
||||
(rt.actors||[]).forEach(a=>{const w=a.worker_id==null?'?':a.worker_id;counts[w]=(counts[w]||0)+1;});
|
||||
const keys=Object.keys(counts).sort();
|
||||
const max=Math.max(1,...Object.values(counts));
|
||||
elWorkers.innerHTML=keys.map(w=>{
|
||||
const c=counts[w];
|
||||
return '<div class="bar-row"><span class="name">worker '+esc(w)+'</span>'+
|
||||
'<div class="bar-track"><div class="bar-fill" style="width:'+(c/max*100).toFixed(1)+'%"></div></div>'+
|
||||
'<span class="num">'+c+'</span></div>';
|
||||
}).join('')||'<div class="empty">no worker placement on frame</div>';
|
||||
}
|
||||
function renderHot(rt){
|
||||
const hot=(rt.actors||[]).slice().sort((a,b)=>(b.mailbox_growth||0)-(a.mailbox_growth||0)).slice(0,8);
|
||||
const gmax=Math.max(1,...hot.map(a=>Math.max(0,a.mailbox_growth||0)));
|
||||
elHot.innerHTML=hot.map(a=>{
|
||||
const g=a.mailbox_growth||0;const gc=growthColor(g);
|
||||
return '<div class="bar-row" data-addr="'+esc(a.address)+'">'+
|
||||
'<span class="name">'+shortAddr(a.address)+(a.name?' '+esc(a.name):'')+'</span>'+
|
||||
'<div class="bar-track"><div class="bar-fill" style="width:'+(Math.max(0,g)/gmax*100).toFixed(1)+'%;background:'+gc+'"></div></div>'+
|
||||
'<span class="num">'+fmt(g,1)+'/s</span></div>';
|
||||
}).join('')||'<div class="empty">no actors</div>';
|
||||
}
|
||||
function renderAlerts(rt){
|
||||
const alerted=(rt.actors||[]).filter(a=>a.poisoned||(a.mailbox_growth||0)>1.5);
|
||||
if(!alerted.length){elAlerts.innerHTML='<div class="empty">fleet healthy</div>';return;}
|
||||
alerted.sort((a,b)=>{const ap=a.poisoned?0:1,bp=b.poisoned?0:1;if(ap!==bp)return ap-bp;return (b.mailbox_growth||0)-(a.mailbox_growth||0);});
|
||||
elAlerts.innerHTML='<table class="table"><tbody>'+
|
||||
alerted.map(a=>{
|
||||
const reason=a.poisoned?'poisoned':'saturating';
|
||||
return '<tr data-addr="'+esc(a.address)+'"><td>'+shortAddr(a.address)+(a.name?' '+esc(a.name):'')+' '+stateBadge(a)+'</td>'+
|
||||
'<td class="muted">'+reason+'</td><td style="text-align:right;color:var(--muted);width:1%;">\u2192</td></tr>';
|
||||
}).join('')+'</tbody></table>';
|
||||
}
|
||||
function colValue(a,key){
|
||||
switch(key){
|
||||
case 'actor':return (a.name||a.address).toLowerCase();
|
||||
case 'type':return typeShort(a.actor_type).toLowerCase();
|
||||
case 'state':return a.poisoned?'poisoned':'running';
|
||||
case 'mailbox':return a.mailbox_depth||0;
|
||||
case 'rate':return a.msg_per_sec||0;
|
||||
case 'processed':return a.messages_processed||0;
|
||||
case 'last':return a.last_msg_type||'';
|
||||
case 'worker':return a.worker_id==null?-1:a.worker_id;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
function visible(rt){
|
||||
let list=rt.actors||[];
|
||||
if(fSearch){const q=fSearch.toLowerCase();list=list.filter(a=>((a.name?a.name+' ':'')+(a.actor_type?typeShort(a.actor_type)+' '+a.actor_type+' ':'')+a.address+' '+(a.worker_id==null?'':a.worker_id)).toLowerCase().indexOf(q)>=0);}
|
||||
if(fState!=='all')list=list.filter(a=>(a.poisoned?'poisoned':'running')===fState);
|
||||
if(fWorker!=='all')list=list.filter(a=>String(a.worker_id)===fWorker);
|
||||
const k=sortKey,dir=sortDir==='asc'?1:-1;
|
||||
return list.slice().sort((x,y)=>{const vx=colValue(x,k),vy=colValue(y,k);let r;if(typeof vx==='number'&&typeof vy==='number')r=vx-vy;else r=String(vx).localeCompare(String(vy));return r*dir;});
|
||||
}
|
||||
function renderHead(){
|
||||
elHead.innerHTML=COLS.map(c=>{
|
||||
const key=c[0],label=c[1];
|
||||
const arrow=sortKey===key?(sortDir==='asc'?' \u25B2':' \u25BC'):'';
|
||||
const order=sortKey===key?(sortDir==='asc'?'ascending':'descending'):'none';
|
||||
return '<th data-key="'+key+'" scope="col" aria-sort="'+order+'">'+label+arrow+'</th>';
|
||||
}).join('');
|
||||
}
|
||||
function populateWorkerFilter(rt){
|
||||
const seen={};(rt.actors||[]).forEach(a=>{if(a.worker_id!=null)seen[a.worker_id]=1;});
|
||||
const keys=Object.keys(seen).sort((a,b)=>a-b);
|
||||
elWorker.innerHTML='<option value="all">all workers</option>'+keys.map(w=>'<option value="'+esc(w)+'">'+esc(w)+'</option>').join('');
|
||||
if(fWorker!=='all'&&!keys.includes(fWorker))fWorker='all';
|
||||
elWorker.value=fWorker;
|
||||
}
|
||||
function renderRows(rt){
|
||||
const list=visible(rt);
|
||||
if(!list.length){elRows.innerHTML='<tr><td colspan="'+COLS.length+'"><div class="empty">No actors match the current filters.</div></td></tr>';return;}
|
||||
elRows.innerHTML=list.map(a=>{
|
||||
const hist=(a.history||[]).map(h=>h.mailbox_depth);
|
||||
const gc=growthColor(a.mailbox_growth||0);
|
||||
const mbox='<span style="display:inline-flex;align-items:center;gap:8px">'+
|
||||
'<span style="color:'+gc+';font-weight:600">'+fmt(a.mailbox_depth)+'</span>'+sparkline(hist,60,16,gc)+'</span>';
|
||||
const actorCell=shortAddr(a.address)+(a.name?'<br><span class="muted" style="font-size:11px">'+esc(a.name)+'</span>':'');
|
||||
const typeCell=a.actor_type?'<span class="muted" title="'+esc(a.actor_type)+'">'+esc(typeShort(a.actor_type))+'</span>':'<span class="muted">\u2014</span>';
|
||||
return '<tr data-addr="'+esc(a.address)+'">'+
|
||||
'<td>'+actorCell+'</td><td>'+typeCell+'</td><td>'+stateBadge(a)+'</td><td>'+mbox+'</td>'+
|
||||
'<td>'+fmt(a.msg_per_sec,1)+'</td><td>'+fmt(a.messages_processed)+'</td>'+
|
||||
'<td><span class="muted" style="display:inline-block;max-width:240px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom">'+esc(a.last_msg_type)+'</span></td>'+
|
||||
'<td>'+(a.worker_id==null?'<span class="muted">\u2014</span>':a.worker_id)+'</td></tr>';
|
||||
}).join('');
|
||||
}
|
||||
function render(rt){
|
||||
if(!rt){[elCards,elDist,elThroughput,elWorkers,elHot,elAlerts,elRows].forEach(e=>e.innerHTML='');return;}
|
||||
elEmpty.hidden=true;
|
||||
renderCards(rt);renderDist(rt);renderThroughput(rt);renderWorkers(rt);renderHot(rt);renderAlerts(rt);
|
||||
populateWorkerFilter(rt);renderHead();renderRows(rt);
|
||||
}
|
||||
async function refresh(){
|
||||
try{
|
||||
const res=await fetch('/api/view/swactor/actor-overview',{cache:'no-store'});
|
||||
if(!res.ok)throw new Error('HTTP '+res.status);
|
||||
const data=await res.json();
|
||||
runtimes=data.runtimes||[];
|
||||
renderRuntimeSelector();
|
||||
const rt=pickRuntime(runtimes);
|
||||
if(rt){elEmpty.hidden=true;}else{elEmpty.className='toast';elEmpty.textContent='No runtime actor frames yet. A live swactor runtime publishing runtime.actors will populate this page.';elEmpty.hidden=false;}
|
||||
render(rt);
|
||||
}catch(e){
|
||||
elEmpty.className='toast err';
|
||||
elEmpty.textContent='Failed to load actor frames: '+e.message;
|
||||
elEmpty.hidden=false;
|
||||
}
|
||||
}
|
||||
function setSort(key){if(sortKey===key)sortDir=sortDir==='asc'?'desc':'asc';else{sortKey=key;sortDir=DEFDIR[key]||'asc';}renderHead();renderRows(pickRuntime(runtimes));}
|
||||
elRuntime.addEventListener('change',e=>{runtimeKey=e.target.value;render(pickRuntime(runtimes));});
|
||||
elSearch.addEventListener('input',e=>{fSearch=e.target.value.toLowerCase();renderRows(pickRuntime(runtimes));});
|
||||
elState.addEventListener('change',e=>{fState=e.target.value;renderRows(pickRuntime(runtimes));});
|
||||
elWorker.addEventListener('change',e=>{fWorker=e.target.value;renderRows(pickRuntime(runtimes));});
|
||||
elHead.addEventListener('click',e=>{const th=e.target.closest('th[data-key]');if(th)setSort(th.getAttribute('data-key'));});
|
||||
document.addEventListener('click',e=>{const el=e.target.closest('[data-addr]');if(el)location.href='/view/swactor/actor-dossier?actor='+encodeURIComponent(el.getAttribute('data-addr'));});
|
||||
renderHead();
|
||||
refresh();
|
||||
setInterval(refresh,POLL_MS);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
635
crates/dashboard/src/swactor/actor_view.rs
Normal file
635
crates/dashboard/src/swactor/actor_view.rs
Normal file
|
|
@ -0,0 +1,635 @@
|
|||
//! Actor-centric read-only view over `runtime.actors` / `runtime.stats` frames.
|
||||
//!
|
||||
//! Pure frame consumer: it folds incoming per-actor snapshots into local state
|
||||
//! and exposes JSON + HTML. It sends nothing back to observed runtimes. Parsing
|
||||
//! is intentionally tolerant of publisher shape — a frame is a flat record that
|
||||
//! may come from the per-worker `DatastreamStatsHook` envelope
|
||||
//! (`{worker_id, actors:[...]}`) or from a process that publishes a merged
|
||||
//! `{actors:[...]}` payload (the dashboard dummy node, app runtimes). Fields the
|
||||
//! publisher includes (name, worker_id, lifecycle flags) are displayed; ones it
|
||||
//! omits are left blank rather than fabricated.
|
||||
//!
|
||||
//! Reliably on the frame today: address, mailbox depth, throughput, last
|
||||
//! message, poisoned flag, and the message-type diet. `worker_id` and `name`
|
||||
//! appear when the publisher sends them. Finer lifecycle granularity
|
||||
//! (new/suspended/stopping), actor Rust type, and spawn age are not on the
|
||||
//! current frame and are therefore not shown — enriching the feed is a separate
|
||||
//! core concern, not a dashboard one.
|
||||
//!
|
||||
//! Two pages share one data model: the fused overview+roster
|
||||
//! (`/view/swactor/actor-overview`) and the per-actor dossier
|
||||
//! (`/view/swactor/actor-dossier`). Each is a self-contained `DashboardView`
|
||||
//! instance ingesting the same channels.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use datastream::frame::{Frame, StreamId};
|
||||
use parking_lot::RwLock;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::swactor::{RUNTIME_ACTORS, RUNTIME_STATS};
|
||||
use crate::view::DashboardView;
|
||||
use crate::{FrameEvent, StreamEvent};
|
||||
|
||||
const CHANNELS: &[&str] = &[RUNTIME_ACTORS, RUNTIME_STATS];
|
||||
const HISTORY_CAP: usize = 512;
|
||||
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(250);
|
||||
const PER_ACTOR_HISTORY_CAP: usize = 120;
|
||||
const GROWTH_WINDOW: usize = 12;
|
||||
const LIVE_TTL: Duration = Duration::from_secs(8);
|
||||
|
||||
/// Read-only actor panel. One instance per served page so each page owns its
|
||||
/// state independently; both ingest the same frames.
|
||||
pub struct ActorPanelView {
|
||||
id: &'static str,
|
||||
title: &'static str,
|
||||
path: &'static str,
|
||||
html: &'static str,
|
||||
state: RwLock<PanelState>,
|
||||
}
|
||||
|
||||
impl ActorPanelView {
|
||||
/// Fused overview + roster at `/view/swactor/actor-overview`.
|
||||
pub fn overview() -> Self {
|
||||
Self::new(
|
||||
"swactor-actor-overview",
|
||||
"Actor overview",
|
||||
"swactor/actor-overview",
|
||||
include_str!("actor_overview.html"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Per-actor dossier at `/view/swactor/actor-dossier`.
|
||||
pub fn dossier() -> Self {
|
||||
Self::new(
|
||||
"swactor-actor-dossier",
|
||||
"Actor dossier",
|
||||
"swactor/actor-dossier",
|
||||
include_str!("actor_dossier.html"),
|
||||
)
|
||||
}
|
||||
|
||||
fn new(id: &'static str, title: &'static str, path: &'static str, html: &'static str) -> Self {
|
||||
Self {
|
||||
id,
|
||||
title,
|
||||
path,
|
||||
html,
|
||||
state: RwLock::new(PanelState::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct PanelState {
|
||||
runtimes: BTreeMap<String, RuntimeState>,
|
||||
}
|
||||
|
||||
struct RuntimeState {
|
||||
stream: StreamEvent,
|
||||
last_seen: Instant,
|
||||
num_workers: Option<u32>,
|
||||
uptime_ms: Option<u64>,
|
||||
actors: BTreeMap<String, ActorState>,
|
||||
history: VecDeque<HistorySample>,
|
||||
}
|
||||
|
||||
impl RuntimeState {
|
||||
fn new(stream: StreamEvent, now: Instant) -> Self {
|
||||
Self {
|
||||
stream,
|
||||
last_seen: now,
|
||||
num_workers: None,
|
||||
uptime_ms: None,
|
||||
actors: BTreeMap::new(),
|
||||
history: VecDeque::with_capacity(HISTORY_CAP),
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, channel: &str, payload: &[u8], now: Instant) {
|
||||
self.last_seen = now;
|
||||
let Ok(value) = serde_json::from_slice::<Value>(payload) else {
|
||||
return;
|
||||
};
|
||||
match channel {
|
||||
RUNTIME_ACTORS => self.apply_actors(&value, now),
|
||||
RUNTIME_STATS => self.apply_stats(&value, now),
|
||||
_ => {}
|
||||
}
|
||||
self.push_history(now);
|
||||
}
|
||||
|
||||
fn apply_actors(&mut self, value: &Value, now: Instant) {
|
||||
// Per-worker `worker_id` wrapper (DatastreamStatsHook shape) is the
|
||||
// default placement for actors that do not carry one inline.
|
||||
let wrapper_worker = u32_field(value, &["worker_id", "worker"]);
|
||||
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
|
||||
for actor in actors {
|
||||
self.apply_actor(actor, now, wrapper_worker);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// A bare actor object per frame (no envelope).
|
||||
self.apply_actor(value, now, wrapper_worker);
|
||||
}
|
||||
|
||||
fn apply_actor(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
|
||||
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
|
||||
return;
|
||||
};
|
||||
let actor = self
|
||||
.actors
|
||||
.entry(address.clone())
|
||||
.or_insert_with(|| ActorState::new(address));
|
||||
actor.apply_json(value, now, default_worker);
|
||||
}
|
||||
|
||||
fn apply_stats(&mut self, value: &Value, now: Instant) {
|
||||
if let Some(num_workers) = u32_field(value, &["num_workers", "workers_live"]) {
|
||||
self.num_workers = Some(num_workers);
|
||||
}
|
||||
if let Some(uptime_ms) = u64_field(value, &["uptime_ms"]) {
|
||||
self.uptime_ms = Some(uptime_ms);
|
||||
}
|
||||
// address -> worker_id mapping; runtime.stats `actors` is [[addr, wid]].
|
||||
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
|
||||
for entry in actors {
|
||||
if let Some(items) = entry.as_array()
|
||||
&& items.len() >= 2
|
||||
&& let (Some(address), Some(worker_id)) =
|
||||
(value_to_string(&items[0]), value_to_u32(&items[1]))
|
||||
{
|
||||
self.actors
|
||||
.entry(address.clone())
|
||||
.or_insert_with(|| ActorState::new(address))
|
||||
.worker_id = Some(worker_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Some publishers carry full per-actor detail under `actor_details`.
|
||||
if let Some(details) = value.get("actor_details").and_then(Value::as_array) {
|
||||
for actor in details {
|
||||
self.apply_actor(actor, now, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_history(&mut self, now: Instant) {
|
||||
let totals = self.totals();
|
||||
if let Some(last) = self.history.back_mut()
|
||||
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
|
||||
{
|
||||
last.mailbox_depth = totals.mailbox_depth;
|
||||
last.msg_per_sec = totals.msg_per_sec;
|
||||
return;
|
||||
}
|
||||
if self.history.len() == HISTORY_CAP {
|
||||
self.history.pop_front();
|
||||
}
|
||||
self.history.push_back(HistorySample {
|
||||
at: now,
|
||||
mailbox_depth: totals.mailbox_depth,
|
||||
msg_per_sec: totals.msg_per_sec,
|
||||
});
|
||||
}
|
||||
|
||||
fn totals(&self) -> Totals {
|
||||
let mut totals = Totals::default();
|
||||
totals.actors = self.actors.len().min(u32::MAX as usize) as u32;
|
||||
for actor in self.actors.values() {
|
||||
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
|
||||
totals.msg_per_sec += actor.msg_per_sec;
|
||||
if actor.poisoned {
|
||||
totals.poisoned = totals.poisoned.saturating_add(1);
|
||||
}
|
||||
}
|
||||
totals
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActorState {
|
||||
address: String,
|
||||
name: Option<String>,
|
||||
actor_type: Option<String>,
|
||||
message_type: Option<String>,
|
||||
worker_id: Option<u32>,
|
||||
mailbox_depth: u32,
|
||||
mailbox_growth: f64,
|
||||
messages_processed: u64,
|
||||
msg_per_sec: f64,
|
||||
last_msg_type: Option<String>,
|
||||
poisoned: bool,
|
||||
message_type_counts: Vec<(String, u64)>,
|
||||
history: VecDeque<ActorHistorySample>,
|
||||
last_update: Option<Instant>,
|
||||
}
|
||||
|
||||
impl ActorState {
|
||||
fn new(address: String) -> Self {
|
||||
Self {
|
||||
address,
|
||||
name: None,
|
||||
actor_type: None,
|
||||
message_type: None,
|
||||
worker_id: None,
|
||||
mailbox_depth: 0,
|
||||
mailbox_growth: 0.0,
|
||||
messages_processed: 0,
|
||||
msg_per_sec: 0.0,
|
||||
last_msg_type: None,
|
||||
poisoned: false,
|
||||
message_type_counts: Vec::new(),
|
||||
history: VecDeque::new(),
|
||||
last_update: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_json(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
|
||||
let elapsed = self
|
||||
.last_update
|
||||
.map(|then| now.duration_since(then).as_secs_f64())
|
||||
.unwrap_or(0.0);
|
||||
if let Some(name) = string_field(value, &["name"]).filter(|name| !name.is_empty()) {
|
||||
self.name = Some(name);
|
||||
}
|
||||
if let Some(actor_type) = string_field(value, &["actor_type"]).filter(|t| !t.is_empty()) {
|
||||
self.actor_type = Some(actor_type);
|
||||
}
|
||||
if let Some(message_type) = string_field(value, &["message_type"]).filter(|t| !t.is_empty()) {
|
||||
self.message_type = Some(message_type);
|
||||
}
|
||||
if let Some(worker_id) = u32_field(value, &["worker_id", "worker"]) {
|
||||
self.worker_id = Some(worker_id);
|
||||
} else if self.worker_id.is_none() {
|
||||
self.worker_id = default_worker;
|
||||
}
|
||||
assign_u32(&mut self.mailbox_depth, value, &["mailbox_depth", "queued"]);
|
||||
assign_u64_rate(
|
||||
&mut self.messages_processed,
|
||||
&mut self.msg_per_sec,
|
||||
value,
|
||||
&["messages_processed", "processed", "messages_handled"],
|
||||
elapsed,
|
||||
);
|
||||
if let Some(last) = string_field(
|
||||
value,
|
||||
&["last_msg_type", "last_message", "last_message_type"],
|
||||
)
|
||||
.filter(|last| !last.is_empty())
|
||||
{
|
||||
self.last_msg_type = Some(last);
|
||||
}
|
||||
if let Some(poisoned) = value.get("poisoned").and_then(Value::as_bool) {
|
||||
self.poisoned = poisoned;
|
||||
}
|
||||
if let Some(counts) = parse_message_type_counts(value.get("message_type_counts")) {
|
||||
self.message_type_counts = counts;
|
||||
}
|
||||
self.last_update = Some(now);
|
||||
self.push_history(now);
|
||||
self.recompute_growth();
|
||||
}
|
||||
|
||||
fn push_history(&mut self, now: Instant) {
|
||||
if let Some(last) = self.history.back_mut()
|
||||
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
|
||||
{
|
||||
last.mailbox_depth = self.mailbox_depth;
|
||||
last.msg_per_sec = self.msg_per_sec;
|
||||
return;
|
||||
}
|
||||
if self.history.len() == PER_ACTOR_HISTORY_CAP {
|
||||
self.history.pop_front();
|
||||
}
|
||||
self.history.push_back(ActorHistorySample {
|
||||
at: now,
|
||||
mailbox_depth: self.mailbox_depth,
|
||||
msg_per_sec: self.msg_per_sec,
|
||||
});
|
||||
}
|
||||
|
||||
/// Mailbox depth/s over the recent window, for trend colouring.
|
||||
fn recompute_growth(&mut self) {
|
||||
let n = self.history.len();
|
||||
if n < 2 {
|
||||
self.mailbox_growth = 0.0;
|
||||
return;
|
||||
}
|
||||
let start = n - n.min(GROWTH_WINDOW);
|
||||
let first = &self.history[start];
|
||||
let last = &self.history[n - 1];
|
||||
let span = last.at.duration_since(first.at).as_secs_f64();
|
||||
self.mailbox_growth = if span > 0.0 {
|
||||
(last.mailbox_depth as f64 - first.mailbox_depth as f64) / span
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct HistorySample {
|
||||
at: Instant,
|
||||
mailbox_depth: u32,
|
||||
msg_per_sec: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ActorHistorySample {
|
||||
at: Instant,
|
||||
mailbox_depth: u32,
|
||||
msg_per_sec: f64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Totals {
|
||||
actors: u32,
|
||||
mailbox_depth: u32,
|
||||
msg_per_sec: f64,
|
||||
poisoned: u32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PanelSnapshot {
|
||||
runtimes: Vec<RuntimeSnapshot>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct RuntimeSnapshot {
|
||||
stream: StreamSnapshot,
|
||||
live: bool,
|
||||
last_seen_ms_ago: u64,
|
||||
summary: SummarySnapshot,
|
||||
actors: Vec<ActorSnapshot>,
|
||||
history: Vec<HistorySnapshot>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct StreamSnapshot {
|
||||
key: String,
|
||||
node: String,
|
||||
life: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SummarySnapshot {
|
||||
actors: u32,
|
||||
msg_per_sec: f64,
|
||||
mailbox_depth: u32,
|
||||
poisoned: u32,
|
||||
uptime_ms: Option<u64>,
|
||||
num_workers: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ActorSnapshot {
|
||||
address: String,
|
||||
name: Option<String>,
|
||||
actor_type: Option<String>,
|
||||
message_type: Option<String>,
|
||||
worker_id: Option<u32>,
|
||||
/// Single derived display state. The frame carries only `poisoned`, so the
|
||||
/// granularity is poisoned | running until the feed is enriched.
|
||||
state: &'static str,
|
||||
mailbox_depth: u32,
|
||||
mailbox_growth: f64,
|
||||
messages_processed: u64,
|
||||
msg_per_sec: f64,
|
||||
last_msg_type: Option<String>,
|
||||
poisoned: bool,
|
||||
message_type_counts: Vec<MessageTypeCountSnapshot>,
|
||||
history: Vec<ActorHistorySnapshot>,
|
||||
last_seen_ms_ago: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct MessageTypeCountSnapshot {
|
||||
message_type: String,
|
||||
count: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct HistorySnapshot {
|
||||
ms_ago: u64,
|
||||
msg_per_sec: f64,
|
||||
mailbox_depth: u32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ActorHistorySnapshot {
|
||||
ms_ago: u64,
|
||||
mailbox_depth: u32,
|
||||
msg_per_sec: f64,
|
||||
}
|
||||
|
||||
impl DashboardView for ActorPanelView {
|
||||
fn id(&self) -> &'static str {
|
||||
self.id
|
||||
}
|
||||
|
||||
fn title(&self) -> &'static str {
|
||||
self.title
|
||||
}
|
||||
|
||||
fn path(&self) -> &'static str {
|
||||
self.path
|
||||
}
|
||||
|
||||
fn channels(&self) -> &'static [&'static str] {
|
||||
CHANNELS
|
||||
}
|
||||
|
||||
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.write();
|
||||
let key = stream_key(&event.stream);
|
||||
state
|
||||
.runtimes
|
||||
.entry(key)
|
||||
.or_insert_with(|| RuntimeState::new(event.stream.clone(), now))
|
||||
.update(&event.channel, &event.payload, now);
|
||||
}
|
||||
|
||||
fn snapshot_json(&self) -> Value {
|
||||
let now = Instant::now();
|
||||
let snapshot = PanelSnapshot {
|
||||
runtimes: self
|
||||
.state
|
||||
.read()
|
||||
.runtimes
|
||||
.values()
|
||||
.map(|runtime| runtime_snapshot(runtime, now))
|
||||
.collect(),
|
||||
};
|
||||
serde_json::to_value(snapshot).unwrap_or_else(|_| json!({ "runtimes": [] }))
|
||||
}
|
||||
|
||||
fn html(&self) -> Option<&'static str> {
|
||||
Some(self.html)
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime_snapshot(runtime: &RuntimeState, now: Instant) -> RuntimeSnapshot {
|
||||
let totals = runtime.totals();
|
||||
RuntimeSnapshot {
|
||||
stream: StreamSnapshot {
|
||||
key: stream_key(&runtime.stream),
|
||||
node: runtime.stream.node.clone(),
|
||||
life: runtime.stream.life,
|
||||
},
|
||||
live: now.duration_since(runtime.last_seen) <= LIVE_TTL,
|
||||
last_seen_ms_ago: now.duration_since(runtime.last_seen).as_millis() as u64,
|
||||
summary: SummarySnapshot {
|
||||
actors: totals.actors,
|
||||
msg_per_sec: totals.msg_per_sec,
|
||||
mailbox_depth: totals.mailbox_depth,
|
||||
poisoned: totals.poisoned,
|
||||
uptime_ms: runtime.uptime_ms,
|
||||
num_workers: runtime.num_workers,
|
||||
},
|
||||
actors: runtime
|
||||
.actors
|
||||
.values()
|
||||
.map(|actor| ActorSnapshot {
|
||||
address: actor.address.clone(),
|
||||
name: actor.name.clone(),
|
||||
actor_type: actor.actor_type.clone(),
|
||||
message_type: actor.message_type.clone(),
|
||||
worker_id: actor.worker_id,
|
||||
state: if actor.poisoned { "poisoned" } else { "running" },
|
||||
mailbox_depth: actor.mailbox_depth,
|
||||
mailbox_growth: actor.mailbox_growth,
|
||||
messages_processed: actor.messages_processed,
|
||||
msg_per_sec: actor.msg_per_sec,
|
||||
last_msg_type: actor.last_msg_type.clone(),
|
||||
poisoned: actor.poisoned,
|
||||
message_type_counts: actor
|
||||
.message_type_counts
|
||||
.iter()
|
||||
.map(|(message_type, count)| MessageTypeCountSnapshot {
|
||||
message_type: message_type.clone(),
|
||||
count: *count,
|
||||
})
|
||||
.collect(),
|
||||
history: actor
|
||||
.history
|
||||
.iter()
|
||||
.map(|sample| ActorHistorySnapshot {
|
||||
ms_ago: now.duration_since(sample.at).as_millis() as u64,
|
||||
mailbox_depth: sample.mailbox_depth,
|
||||
msg_per_sec: sample.msg_per_sec,
|
||||
})
|
||||
.collect(),
|
||||
last_seen_ms_ago: actor
|
||||
.last_update
|
||||
.map(|then| now.duration_since(then).as_millis() as u64)
|
||||
.unwrap_or(0),
|
||||
})
|
||||
.collect(),
|
||||
history: runtime
|
||||
.history
|
||||
.iter()
|
||||
.map(|sample| HistorySnapshot {
|
||||
ms_ago: now.duration_since(sample.at).as_millis() as u64,
|
||||
msg_per_sec: sample.msg_per_sec,
|
||||
mailbox_depth: sample.mailbox_depth,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// --- tolerant JSON helpers (publisher-shape-agnostic readers) ---------------
|
||||
|
||||
fn assign_u32(slot: &mut u32, value: &Value, names: &[&str]) {
|
||||
if let Some(v) = u32_field(value, names) {
|
||||
*slot = v;
|
||||
}
|
||||
}
|
||||
|
||||
fn assign_u64_rate(slot: &mut u64, rate: &mut f64, value: &Value, names: &[&str], elapsed: f64) {
|
||||
if let Some(next) = u64_field(value, names) {
|
||||
if elapsed > 0.0 && next > *slot {
|
||||
*rate = (next - *slot) as f64 / elapsed;
|
||||
} else if next < *slot {
|
||||
*rate = 0.0;
|
||||
}
|
||||
*slot = next;
|
||||
}
|
||||
}
|
||||
|
||||
fn u32_field(value: &Value, names: &[&str]) -> Option<u32> {
|
||||
u64_field(value, names).and_then(|v| u32::try_from(v).ok())
|
||||
}
|
||||
|
||||
fn u64_field(value: &Value, names: &[&str]) -> Option<u64> {
|
||||
names
|
||||
.iter()
|
||||
.find_map(|name| value.get(*name).and_then(value_to_u64))
|
||||
}
|
||||
|
||||
fn string_field(value: &Value, names: &[&str]) -> Option<String> {
|
||||
names
|
||||
.iter()
|
||||
.find_map(|name| value.get(*name).and_then(value_to_string))
|
||||
}
|
||||
|
||||
fn value_to_u32(value: &Value) -> Option<u32> {
|
||||
value_to_u64(value).and_then(|v| u32::try_from(v).ok())
|
||||
}
|
||||
|
||||
fn value_to_u64(value: &Value) -> Option<u64> {
|
||||
value
|
||||
.as_u64()
|
||||
.or_else(|| value.as_str().and_then(|s| s.parse::<u64>().ok()))
|
||||
}
|
||||
|
||||
fn value_to_string(value: &Value) -> Option<String> {
|
||||
value.as_str().map(ToOwned::to_owned).or_else(|| {
|
||||
if value.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(value.to_string())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Accepts pair form `["Type", N]`, object form `{"ty": "Type", "count": N}`,
|
||||
/// and map form `{"Type": N}`.
|
||||
fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>> {
|
||||
let value = value?;
|
||||
if let Some(items) = value.as_array() {
|
||||
let mut out = Vec::new();
|
||||
for item in items {
|
||||
if let Some(pair) = item.as_array()
|
||||
&& pair.len() >= 2
|
||||
&& let (Some(name), Some(count)) =
|
||||
(value_to_string(&pair[0]), value_to_u64(&pair[1]))
|
||||
{
|
||||
out.push((name, count));
|
||||
continue;
|
||||
}
|
||||
if let Some(name) = string_field(item, &["ty", "message_type", "type", "name"])
|
||||
&& let Some(count) = u64_field(item, &["count"])
|
||||
{
|
||||
out.push((name, count));
|
||||
}
|
||||
}
|
||||
return Some(out);
|
||||
}
|
||||
if let Some(map) = value.as_object() {
|
||||
let mut out: Vec<(String, u64)> = map
|
||||
.iter()
|
||||
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
|
||||
.collect();
|
||||
out.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
return Some(out);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn stream_key(stream: &StreamEvent) -> String {
|
||||
format!("{}#{}", stream.node, stream.life)
|
||||
}
|
||||
|
|
@ -2,10 +2,12 @@ use std::sync::Arc;
|
|||
|
||||
use crate::view::DashboardView;
|
||||
|
||||
mod actor_view;
|
||||
mod worker_page;
|
||||
mod worker_view;
|
||||
|
||||
pub use worker_view::SwactorWorkerView;
|
||||
pub use actor_view::ActorPanelView;
|
||||
|
||||
pub const RUNTIME_STATS: &str = "runtime.stats";
|
||||
pub const RUNTIME_WORKERS: &str = "runtime.workers";
|
||||
|
|
@ -14,3 +16,13 @@ pub const RUNTIME_ACTORS: &str = "runtime.actors";
|
|||
pub fn worker_view() -> Arc<dyn DashboardView> {
|
||||
Arc::new(SwactorWorkerView::default())
|
||||
}
|
||||
|
||||
/// Built-in actor overview + roster view (`/view/swactor/actor-overview`).
|
||||
pub fn actor_overview_view() -> Arc<dyn DashboardView> {
|
||||
Arc::new(ActorPanelView::overview())
|
||||
}
|
||||
|
||||
/// Built-in actor dossier view (`/view/swactor/actor-dossier`).
|
||||
pub fn actor_dossier_view() -> Arc<dyn DashboardView> {
|
||||
Arc::new(ActorPanelView::dossier())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -758,9 +758,11 @@ impl<'a> RuntimeActorStatsRecord<'a> {
|
|||
actors: snapshots
|
||||
.iter()
|
||||
.map(|snapshot| RuntimeActorSnapshotRecord {
|
||||
address: snapshot.address.to_string(),
|
||||
address: snapshot.address.to_full_hex(),
|
||||
mailbox_depth: snapshot.mailbox_depth,
|
||||
last_msg_type: snapshot.last_msg_type,
|
||||
actor_type: snapshot.actor_type,
|
||||
message_type: snapshot.message_type,
|
||||
messages_processed: snapshot.messages_processed,
|
||||
poisoned: snapshot.poisoned,
|
||||
message_type_counts: snapshot
|
||||
|
|
@ -779,6 +781,8 @@ struct RuntimeActorSnapshotRecord<'a> {
|
|||
address: String,
|
||||
mailbox_depth: usize,
|
||||
last_msg_type: Option<&'static str>,
|
||||
actor_type: Option<&'static str>,
|
||||
message_type: Option<&'static str>,
|
||||
messages_processed: u64,
|
||||
poisoned: bool,
|
||||
message_type_counts: Vec<RuntimeMessageTypeCount<'a>>,
|
||||
|
|
|
|||
|
|
@ -300,6 +300,8 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
|||
address: actor,
|
||||
mailbox_depth: 3,
|
||||
last_msg_type: Some("Ping"),
|
||||
actor_type: Some("TestActor"),
|
||||
message_type: Some("Ping"),
|
||||
messages_processed: 5,
|
||||
poisoned: false,
|
||||
message_type_counts: vec![("Ping", 5)],
|
||||
|
|
@ -313,11 +315,13 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
|||
assert_eq!(delivery.channel.channel, runtime);
|
||||
let json: Value = serde_json::from_slice(&delivery.payload).unwrap();
|
||||
assert_eq!(json["worker_id"], 2);
|
||||
assert_eq!(json["actors"][0]["address"], actor.to_string());
|
||||
assert_eq!(json["actors"][0]["address"], actor.to_full_hex());
|
||||
assert_eq!(json["actors"][0]["mailbox_depth"], 3);
|
||||
assert_eq!(json["actors"][0]["last_msg_type"], "Ping");
|
||||
assert_eq!(json["actors"][0]["messages_processed"], 5);
|
||||
assert_eq!(json["actors"][0]["message_type_counts"][0]["ty"], "Ping");
|
||||
assert_eq!(json["actors"][0]["actor_type"], "TestActor");
|
||||
assert_eq!(json["actors"][0]["message_type"], "Ping");
|
||||
}
|
||||
|
||||
fn positions(events: &[DatastreamEvent]) -> Vec<u64> {
|
||||
|
|
|
|||
|
|
@ -19,9 +19,8 @@ iroh = "0.98"
|
|||
# `test-utils` exposes `CaRootsConfig::insecure_skip_verify()` so clients can
|
||||
# trust operator-controlled custom relays with self-signed QAD certs.
|
||||
iroh-relay = { version = "0.98", features = ["test-utils"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "time"] }
|
||||
tokio.workspace = true
|
||||
parking_lot = "0.12"
|
||||
|
||||
[dev-dependencies]
|
||||
iroh-relay = { version = "0.98", features = ["server", "test-utils"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "io-util", "time"] }
|
||||
|
|
|
|||
2
rust-toolchain.toml
Normal file
2
rust-toolchain.toml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
[toolchain]
|
||||
channel = "nightly-2026-02-07"
|
||||
12
src/actor.rs
12
src/actor.rs
|
|
@ -162,6 +162,18 @@ impl ActorAddress {
|
|||
crate::get_random(&mut bytes);
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
/// Full 64-character hex encoding of all 32 bytes, for displays that need
|
||||
/// the untruncated address (dashboards). [`Display`](std::fmt::Display)
|
||||
/// stays short for logs.
|
||||
pub fn to_full_hex(&self) -> String {
|
||||
use std::fmt::Write;
|
||||
let mut s = String::with_capacity(64);
|
||||
for b in &self.0 {
|
||||
let _ = write!(s, "{:02x}", b);
|
||||
}
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Environment ─────────────────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -124,6 +124,8 @@ pub struct ActorSnapshot {
|
|||
pub address: ActorAddress,
|
||||
pub mailbox_depth: usize,
|
||||
pub last_msg_type: Option<&'static str>,
|
||||
pub actor_type: Option<&'static str>,
|
||||
pub message_type: Option<&'static str>,
|
||||
pub messages_processed: u64,
|
||||
pub poisoned: bool,
|
||||
/// Per-message-type counts, sorted descending by count.
|
||||
|
|
|
|||
|
|
@ -998,10 +998,13 @@ impl ActorPool {
|
|||
let mut type_counts: Vec<(&'static str, u64)> =
|
||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
let metadata = slot.actor.metadata();
|
||||
ActorSnapshot {
|
||||
address: addr,
|
||||
mailbox_depth: slot.mailbox.len(),
|
||||
last_msg_type: slot.last_msg_type,
|
||||
actor_type: Some(metadata.actor_type_name),
|
||||
message_type: Some(metadata.message_type_name),
|
||||
messages_processed: slot.messages_processed,
|
||||
poisoned: slot.poisoned,
|
||||
message_type_counts: type_counts,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ publish = false
|
|||
reqwest = { version = "0.12", features = ["json"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tokio = { version = "1", features = ["macros", "rt", "time"] }
|
||||
tokio.workspace = true
|
||||
urlencoding = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ OPTIONS:
|
|||
Select the runtime provider
|
||||
--config <path> Load config overlay
|
||||
--pipeline-stages <count> Number of pipeline stages
|
||||
--cached-model[=<path>] Use discovered or explicit cached GGUF model
|
||||
--cached-model[=<path>] Use discovered or explicit cached GGUF model (default for --process)
|
||||
--dump-logs[=<path>] Write datastream frame log
|
||||
--run-id <id> Override run id
|
||||
--skip-rebuild Reuse existing Cargo artifacts
|
||||
|
|
|
|||
Loading…
Reference in a new issue